ralsina/tartrazine
1
0
Fork 0
mirror of https://github.com/ralsina/tartrazine.git synced 2024-11-12 22:42:23 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #23 from mcarmonaa/filenames

Browse Source 
Added language detection by filename strategy
This commit is contained in:
Santiago M. Mola 2017-05-04 17:24:18 +02:00 committed by GitHub
commit 1cb715ae26
26 changed files with 499 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cli/slinguist/main.go
View File

@ -48,7 +48,7 @@ func main() {
return nil
}
l := slinguist.GetLanguage(path, content)
l := slinguist.GetLanguage(filepath.Base(path), content)
r, err := filepath.Rel(root, path)
if err != nil {

4
common.go
View File

@ -36,6 +36,10 @@ func GetLanguageExtensions(language string) []string {
// GetLanguage return the Language for a given filename and file content.
func GetLanguage(filename string, content []byte) string {
if lang, safe := GetLanguageByFilename(filename); safe {
return lang
}
if lang, safe := GetLanguageByShebang(content); safe {
return lang
}

0
languages.go → extensions_map.go
View File

10
filename.go Normal file
View File

@ -0,0 +1,10 @@
package slinguist
func GetLanguageByFilename(filename string) (lang string, safe bool) {
lang, safe = languagesByFilename[filename]
if lang == "" {
lang = OtherLanguage
}
return
}

37
filename_test.go Normal file
View File

@ -0,0 +1,37 @@
package slinguist
import . "gopkg.in/check.v1"
func (s *TSuite) TestGetLanguageByFilename(c *C) {
lang, safe := GetLanguageByFilename(`unknown.interpreter`)
c.Assert(lang, Equals, OtherLanguage)
c.Assert(safe, Equals, false)
lang, safe = GetLanguageByFilename(`.bashrc`)
c.Assert(lang, Equals, "Shell")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByFilename(`Dockerfile`)
c.Assert(lang, Equals, "Dockerfile")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByFilename(`Makefile.frag`)
c.Assert(lang, Equals, "Makefile")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByFilename(`makefile`)
c.Assert(lang, Equals, "Makefile")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByFilename(`Vagrantfile`)
c.Assert(lang, Equals, "Ruby")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByFilename(`_vimrc`)
c.Assert(lang, Equals, "Vim script")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByFilename(`pom.xml`)
c.Assert(lang, Equals, "Maven POM")
c.Assert(safe, Equals, true)
}

140
filenames_map.go Normal file
View File

@ -0,0 +1,140 @@
package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: dae33dc2b20cddc85d1300435c3be7118a7115a9
var languagesByFilename = map[string]string{
".Rprofile": "R",
".XCompose": "XCompose",
".abbrev_defs": "Emacs Lisp",
".arcconfig": "JSON",
".babelrc": "JSON5",
".bash_history": "Shell",
".bash_logout": "Shell",
".bash_profile": "Shell",
".bashrc": "Shell",
".clang-format": "YAML",
".classpath": "XML",
".emacs": "Emacs Lisp",
".emacs.desktop": "Emacs Lisp",
".factor-boot-rc": "Factor",
".factor-rc": "Factor",
".gclient": "Python",
".gnus": "Emacs Lisp",
".jshintrc": "JSON",
".nvimrc": "Vim script",
".php_cs": "PHP",
".php_cs.dist": "PHP",
".project": "XML",
".pryrc": "Ruby",
".spacemacs": "Emacs Lisp",
".vimrc": "Vim script",
".viper": "Emacs Lisp",
"APKBUILD": "Alpine Abuild",
"App.config": "XML",
"Appraisals": "Ruby",
"BSDmakefile": "Makefile",
"BUCK": "Python",
"BUILD": "Python",
"Berksfile": "Ruby",
"Brewfile": "Ruby",
"Buildfile": "Ruby",
"CMakeLists.txt": "CMake",
"COPYING": "Text",
"COPYRIGHT.regex": "Text",
"Cakefile": "CoffeeScript",
"Cask": "Emacs Lisp",
"Dangerfile": "Ruby",
"Deliverfile": "Ruby",
"Dockerfile": "Dockerfile",
"Emakefile": "Erlang",
"FONTLOG": "Text",
"Fakefile": "Fancy",
"Fastfile": "Ruby",
"GNUmakefile": "Makefile",
"Gemfile": "Ruby",
"Gemfile.lock": "Ruby",
"Guardfile": "Ruby",
"INSTALL": "Text",
"INSTALL.mysql": "Text",
"Jakefile": "JavaScript",
"Jarfile": "Ruby",
"Jenkinsfile": "Groovy",
"Kbuild": "Makefile",
"LICENSE": "Text",
"LICENSE.mysql": "Text",
"Makefile": "Makefile",
"Makefile.am": "Makefile",
"Makefile.boot": "Makefile",
"Makefile.frag": "Makefile",
"Makefile.in": "Makefile",
"Makefile.inc": "Makefile",
"Mavenfile": "Ruby",
"Modulefile": "Puppet",
"NEWS": "Text",
"Notebook": "Jupyter Notebook",
"NuGet.config": "XML",
"Nukefile": "Nu",
"PKGBUILD": "Shell",
"Phakefile": "PHP",
"Podfile": "Ruby",
"Project.ede": "Emacs Lisp",
"Puppetfile": "Ruby",
"README.1ST": "Text",
"README.me": "Text",
"README.mysql": "Text",
"ROOT": "Isabelle ROOT",
"Rexfile": "Perl6",
"SConscript": "Python",
"SConstruct": "Python",
"Settings.StyleCop": "XML",
"Slakefile": "LiveScript",
"Snakefile": "Python",
"Snapfile": "Ruby",
"Thorfile": "Ruby",
"Vagrantfile": "Ruby",
"WORKSPACE": "Python",
"Web.Debug.config": "XML",
"Web.Release.config": "XML",
"Web.config": "XML",
"XCompose": "XCompose",
"_emacs": "Emacs Lisp",
"_vimrc": "Vim script",
"abbrev_defs": "Emacs Lisp",
"ant.xml": "Ant Build System",
"build.xml": "Ant Build System",
"buildfile": "Ruby",
"click.me": "Text",
"composer.lock": "JSON",
"configure.ac": "M4Sugar",
"delete.me": "Text",
"descrip.mmk": "Module Management System",
"descrip.mms": "Module Management System",
"gradlew": "Shell",
"gvimrc": "Vim script",
"keep.me": "Text",
"ld.script": "Linker Script",
"makefile": "Makefile",
"makefile.sco": "Makefile",
"mcmod.info": "JSON",
"meson.build": "Meson",
"meson_options.txt": "Meson",
"mix.lock": "Elixir",
"mkfile": "Makefile",
"mmn": "Roff",
"mmt": "Roff",
"nginx.conf": "Nginx",
"nvimrc": "Vim script",
"packages.config": "XML",
"pom.xml": "Maven POM",
"read.me": "Text",
"rebar.config": "Erlang",
"rebar.config.lock": "Erlang",
"rebar.lock": "Erlang",
"riemann.config": "Clojure",
"test.me": "Text",
"vimrc": "Vim script",
"wscript": "Python",
"xcompose": "XCompose",
}

0
internal/code-generator/assets/languages.go.tmpl → internal/code-generator/assets/extensions.go.tmpl
View File

10
internal/code-generator/generator/test_files/vendor.test.go.tmpl → internal/code-generator/assets/filenames.go.tmpl
View File

@ -4,10 +4,8 @@ package slinguist
// THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: {{ getCommit }}
import "gopkg.in/toqueteos/substring.v1"
var vendorMatchers = substring.Or(
{{range $regexp := . -}}
substring.Regexp(`{{ $regexp }}`),
var languagesByFilename = map[string]string{
{{range $filename, $language := . -}}
"{{ $filename }}": {{- printf "%q" $language -}},
{{end -}}
)
}

2
internal/code-generator/generator/documentation.go
View File

@ -16,7 +16,7 @@ func Documentation(data []byte, documentationTmplPath, documentationTmplName, co
}
buf := &bytes.Buffer{}
if err := executeVendorTemplate(buf, regexpList, documentationTmplPath, documentationTmplName, commit); err != nil {
if err := executeDocumentationTemplate(buf, regexpList, documentationTmplPath, documentationTmplName, commit); err != nil {
return nil, err
}

29
internal/code-generator/generator/langs.go → internal/code-generator/generator/extensions.go
View File

@ -3,21 +3,14 @@ package generator
import (
"bytes"
"io"
"sort"
"strings"
"text/template"
"gopkg.in/yaml.v2"
yaml "gopkg.in/yaml.v2"
)
type languageInfo struct {
Type string `yaml:"type,omitempty"`
Extensions []string `yaml:"extensions,omitempty,flow"`
Interpreters []string `yaml:"interpreters,omitempty,flow"`
}
// Languages reads from buf and builds languages.go file from languagesTmplPath.
func Languages(data []byte, languagesTmplPath, languagesTmplName, commit string) ([]byte, error) {
// Extensions reads from buf and builds extensions_map.go file from extensionsTmplPath.
func Extensions(data []byte, extensionsTmplPath, extensionsTmplName, commit string) ([]byte, error) {
languages := make(map[string]*languageInfo)
if err := yaml.Unmarshal(data, &languages); err != nil {
return nil, err
@ -27,23 +20,13 @@ func Languages(data []byte, languagesTmplPath, languagesTmplName, commit string)
languagesByExtension := buildExtensionLanguageMap(languages, orderedKeyList)
buf := &bytes.Buffer{}
if err := executeLanguagesTemplate(buf, languagesByExtension, languagesTmplPath, languagesTmplName, commit); err != nil {
if err := executeExtensionsTemplate(buf, languagesByExtension, extensionsTmplPath, extensionsTmplName, commit); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func getAlphabeticalOrderedKeys(languages map[string]*languageInfo) []string {
keyList := make([]string, 0)
for lang := range languages {
keyList = append(keyList, lang)
}
sort.Strings(keyList)
return keyList
}
func buildExtensionLanguageMap(languages map[string]*languageInfo, orderedKeyList []string) map[string][]string {
extensionLangsMap := make(map[string][]string)
for _, lang := range orderedKeyList {
@ -56,13 +39,13 @@ func buildExtensionLanguageMap(languages map[string]*languageInfo, orderedKeyLis
return extensionLangsMap
}
func executeLanguagesTemplate(out io.Writer, languagesByExtension map[string][]string, languagesTmplPath, languagesTmpl, commit string) error {
func executeExtensionsTemplate(out io.Writer, languagesByExtension map[string][]string, extensionsTmplPath, extensionsTmpl, commit string) error {
fmap := template.FuncMap{
"getCommit": func() string { return commit },
"formatStringSlice": func(slice []string) string { return `"` + strings.Join(slice, `","`) + `"` },
}
t := template.Must(template.New(languagesTmpl).Funcs(fmap).ParseFiles(languagesTmplPath))
t := template.Must(template.New(extensionsTmpl).Funcs(fmap).ParseFiles(extensionsTmplPath))
if err := t.Execute(out, languagesByExtension); err != nil {
return err
}

50
internal/code-generator/generator/filenames.go Normal file
View File

@ -0,0 +1,50 @@
package generator
import (
"bytes"
"io"
"text/template"
yaml "gopkg.in/yaml.v2"
)
// Filenames reads from buf and builds filenames_map.go file from filenamesTmplPath.
func Filenames(data []byte, filenamesTmplPath, filenamesTmplName, commit string) ([]byte, error) {
languages := make(map[string]*languageInfo)
if err := yaml.Unmarshal(data, &languages); err != nil {
return nil, err
}
languagesByFilename := buildFilenameLanguageMap(languages)
buf := &bytes.Buffer{}
if err := executeFilenamesTemplate(buf, languagesByFilename, filenamesTmplPath, filenamesTmplName, commit); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func buildFilenameLanguageMap(languages map[string]*languageInfo) map[string]string {
filenameLangMap := make(map[string]string)
for lang, langInfo := range languages {
for _, filename := range langInfo.Filenames {
filenameLangMap[filename] = lang
}
}
return filenameLangMap
}
func executeFilenamesTemplate(out io.Writer, languagesByFilename map[string]string, filenamesTmplPath, filenamesTmpl, commit string) error {
fmap := template.FuncMap{
"getCommit": func() string { return commit },
}
t := template.Must(template.New(filenamesTmpl).Funcs(fmap).ParseFiles(filenamesTmplPath))
if err := t.Execute(out, languagesByFilename); err != nil {
return err
}
return nil
}

127
internal/code-generator/generator/generator_test.go
View File

@ -13,164 +13,137 @@ const (
commitTest = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7"
// Languages test
ymlTestFile = "test_files/languages.test.yml"
langGold = "test_files/languages.gold"
languagesTestTmplPath = "test_files/languages.test.go.tmpl"
languagesTestTmplName = "languages.test.go.tmpl"
extensionsTestFile = "test_files/extensions.test.yml"
extensionsGold = "test_files/extensions.gold"
extensionsTestTmplPath = "../assets/extensions.go.tmpl"
extensionsTestTmplName = "extensions.go.tmpl"
// Heuristics test
heuristicsTestFile = "test_files/heuristics.test.rb"
contentGold = "test_files/content.gold"
contentTestTmplPath = "test_files/content.test.go.tmpl"
contentTestTmplName = "content.test.go.tmpl"
contentTestTmplPath = "../assets/content.go.tmpl"
contentTestTmplName = "content.go.tmpl"
// Vendor test
vendorTestFile = "test_files/vendor.test.yml"
vendorGold = "test_files/vendor.gold"
vendorTestTmplPath = "test_files/vendor.test.go.tmpl"
vendorTestTmplName = "vendor.test.go.tmpl"
vendorTestTmplPath = "../assets/vendor.go.tmpl"
vendorTestTmplName = "vendor.go.tmpl"
// Documentation test
documentationTestFile = "test_files/documentation.test.yml"
documentationGold = "test_files/documentation.gold"
documentationTestTmplPath = "test_files/documentation.test.go.tmpl"
documentationTestTmplName = "documentation.test.go.tmpl"
documentationTestTmplPath = "../assets/documentation.go.tmpl"
documentationTestTmplName = "documentation.go.tmpl"
// Types test
typesTestFile = "test_files/type.test.yml"
typesGold = "test_files/type.gold"
typesTestTmplPath = "test_files/type.test.go.tmpl"
typesTestTmplName = "type.test.go.tmpl"
typesTestTmplPath = "../assets/type.go.tmpl"
typesTestTmplName = "type.go.tmpl"
// Interpreters test
interpretersTestFile = "test_files/interpreters.test.yml"
interpretersGold = "test_files/interpreters.gold"
interpretersTestTmplPath = "test_files/interpreters.test.go.tmpl"
interpretersTestTmplName = "interpreters.test.go.tmpl"
interpretersTestTmplPath = "../assets/interpreters.go.tmpl"
interpretersTestTmplName = "interpreters.go.tmpl"
// Filenames test
filenamesTestFile = "test_files/filenames.test.yml"
filenamesGold = "test_files/filenames.gold"
filenamesTestTmplPath = "../assets/filenames.go.tmpl"
filenamesTestTmplName = "filenames.go.tmpl"
)
func TestFromFile(t *testing.T) {
goldLang, err := ioutil.ReadFile(langGold)
assert.NoError(t, err)
goldContent, err := ioutil.ReadFile(contentGold)
assert.NoError(t, err)
goldVendor, err := ioutil.ReadFile(vendorGold)
assert.NoError(t, err)
goldDocumentation, err := ioutil.ReadFile(documentationGold)
assert.NoError(t, err)
goldTypes, err := ioutil.ReadFile(typesGold)
assert.NoError(t, err)
goldInterpreters, err := ioutil.ReadFile(interpretersGold)
assert.NoError(t, err)
outPathLang, err := ioutil.TempFile("/tmp", "generator-test-")
assert.NoError(t, err)
defer os.Remove(outPathLang.Name())
outPathContent, err := ioutil.TempFile("/tmp", "generator-test-")
assert.NoError(t, err)
defer os.Remove(outPathContent.Name())
outPathVendor, err := ioutil.TempFile("/tmp", "generator-test-")
assert.NoError(t, err)
defer os.Remove(outPathVendor.Name())
outPathDocumentation, err := ioutil.TempFile("/tmp", "generator-test-")
assert.NoError(t, err)
defer os.Remove(outPathDocumentation.Name())
outPathTypes, err := ioutil.TempFile("/tmp", "generator-test-")
assert.NoError(t, err)
defer os.Remove(outPathTypes.Name())
outPathInterpreters, err := ioutil.TempFile("/tmp", "generator-test-")
assert.NoError(t, err)
defer os.Remove(outPathInterpreters.Name())
tests := []struct {
name string
fileToParse string
outPath string
tmplPath string
tmplName string
commit string
generate Func
wantOut []byte
wantOut string
}{
{
name: "TestFromFile_Language",
fileToParse: ymlTestFile,
outPath: outPathLang.Name(),
tmplPath: languagesTestTmplPath,
tmplName: languagesTestTmplName,
fileToParse: extensionsTestFile,
tmplPath: extensionsTestTmplPath,
tmplName: extensionsTestTmplName,
commit: commitTest,
generate: Languages,
wantOut: goldLang,
generate: Extensions,
wantOut: extensionsGold,
},
{
name: "TestFromFile_Heuristics",
fileToParse: heuristicsTestFile,
outPath: outPathContent.Name(),
tmplPath: contentTestTmplPath,
tmplName: contentTestTmplName,
commit: commitTest,
generate: Heuristics,
wantOut: goldContent,
wantOut: contentGold,
},
{
name: "TestFromFile_Vendor",
fileToParse: vendorTestFile,
outPath: outPathVendor.Name(),
tmplPath: vendorTestTmplPath,
tmplName: vendorTestTmplName,
commit: commitTest,
generate: Vendor,
wantOut: goldVendor,
wantOut: vendorGold,
},
{
name: "TestFromFile_Documentation",
fileToParse: documentationTestFile,
outPath: outPathDocumentation.Name(),
tmplPath: documentationTestTmplPath,
tmplName: documentationTestTmplName,
commit: commitTest,
generate: Documentation,
wantOut: goldDocumentation,
wantOut: documentationGold,
},
{
name: "TestFromFile_Types",
fileToParse: typesTestFile,
outPath: outPathTypes.Name(),
tmplPath: typesTestTmplPath,
tmplName: typesTestTmplName,
commit: commitTest,
generate: Types,
wantOut: goldTypes,
wantOut: typesGold,
},
{
name: "TestFromFile_Interpreters",
fileToParse: interpretersTestFile,
outPath: outPathInterpreters.Name(),
tmplPath: interpretersTestTmplPath,
tmplName: interpretersTestTmplName,
commit: commitTest,
generate: Interpreters,
wantOut: goldInterpreters,
wantOut: interpretersGold,
},
{
name: "TestFromFile_Filenames",
fileToParse: filenamesTestFile,
tmplPath: filenamesTestTmplPath,
tmplName: filenamesTestTmplName,
commit: commitTest,
generate: Filenames,
wantOut: filenamesGold,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := FromFile(tt.fileToParse, tt.outPath, tt.tmplPath, tt.tmplName, tt.commit, tt.generate)
gold, err := ioutil.ReadFile(tt.wantOut)
assert.NoError(t, err)
out, err := ioutil.ReadFile(tt.outPath)
outPath, err := ioutil.TempFile("/tmp", "generator-test-")
assert.NoError(t, err)
assert.EqualValues(t, tt.wantOut, out, fmt.Sprintf("FromFile() = %v, want %v", string(out), string(tt.wantOut)))
defer os.Remove(outPath.Name())
err = FromFile(tt.fileToParse, outPath.Name(), tt.tmplPath, tt.tmplName, tt.commit, tt.generate)
assert.NoError(t, err)
out, err := ioutil.ReadFile(outPath.Name())
assert.NoError(t, err)
assert.EqualValues(t, gold, out, fmt.Sprintf("FromFile() = %v, want %v", string(out), string(tt.wantOut)))
})
}
}

4
internal/code-generator/generator/heuristics.go
View File

@ -299,7 +299,7 @@ func getHeuristics(line string) []*heuristic {
}
if reg != "" {
reg = convToValidRegexp(reg)
reg = convertToValidRegexp(reg)
heuristics = append(heuristics, &heuristic{Regexp: reg})
}
}
@ -327,7 +327,7 @@ func replaceRegexpVariables(reg string) string {
return repl
}
func convToValidRegexp(reg string) string {
func convertToValidRegexp(reg string) string {
// example: `/^(\s*)(<Project|<Import|<Property|<?xml|xmlns)/i``
// Ruby modifier "m" matches multiple lines, recognizing newlines as normal characters, Go use flag "s" for that.
const (

20
internal/code-generator/generator/langinfo.go Normal file
View File

@ -0,0 +1,20 @@
package generator
import "sort"
type languageInfo struct {
Type string `yaml:"type,omitempty"`
Extensions []string `yaml:"extensions,omitempty,flow"`
Interpreters []string `yaml:"interpreters,omitempty,flow"`
Filenames []string `yaml:"filenames,omitempty,flow"`
}
func getAlphabeticalOrderedKeys(languages map[string]*languageInfo) []string {
keyList := make([]string, 0)
for lang := range languages {
keyList = append(keyList, lang)
}
sort.Strings(keyList)
return keyList
}

97
internal/code-generator/generator/test_files/content.gold
View File

@ -34,6 +34,53 @@ var matchers = map[string]languageMatcher{
return OtherLanguage, false
},
".f": func(i []byte) (string, bool) {
if f_Forth_Matcher_0.Match(i) {
return "Forth", true
} else if f_FilebenchWML_Matcher_0.Match(i) {
return "Filebench WML", true
} else if f_FORTRAN_Matcher_0.Match(i) {
return "FORTRAN", true
}
return OtherLanguage, false
},
".h": func(i []byte) (string, bool) {
if h_ObjectiveDashC_Matcher_0.Match(i) {
return "Objective-C", true
} else if h_CPlusPlus_Matcher_0.Match(i) || h_CPlusPlus_Matcher_1.Match(i) || h_CPlusPlus_Matcher_2.Match(i) || h_CPlusPlus_Matcher_3.Match(i) || h_CPlusPlus_Matcher_4.Match(i) || h_CPlusPlus_Matcher_5.Match(i) || h_CPlusPlus_Matcher_6.Match(i) {
return "C++", true
}
return OtherLanguage, false
},
".lsp": func(i []byte) (string, bool) {
if lsp_CommonLisp_Matcher_0.Match(i) {
return "Common Lisp", true
} else if lsp_NewLisp_Matcher_0.Match(i) {
return "NewLisp", true
}
return OtherLanguage, false
},
".lisp": func(i []byte) (string, bool) {
if lisp_CommonLisp_Matcher_0.Match(i) {
return "Common Lisp", true
} else if lisp_NewLisp_Matcher_0.Match(i) {
return "NewLisp", true
}
return OtherLanguage, false
},
".md": func(i []byte) (string, bool) {
if md_Markdown_Matcher_0.Match(i) || md_Markdown_Matcher_1.Match(i) {
return "Markdown", true
} else if md_GCCmachinedescription_Matcher_0.Match(i) {
return "GCC machine description", true
}
return "Markdown", true
},
".ms": func(i []byte) (string, bool) {
if ms_Groff_Matcher_0.Match(i) {
return "Groff", true
@ -63,19 +110,45 @@ var matchers = map[string]languageMatcher{
return OtherLanguage, false
},
".rpy": func(i []byte) (string, bool) {
if rpy_Python_Matcher_0.Match(i) {
return "Python", true
}
return "Ren'Py", true
},
}
var (
asc_PublicKey_Matcher_0 = regexp.MustCompile(`(?m)^(----[- ]BEGIN|ssh-(rsa|dss)) `)
asc_AsciiDoc_Matcher_0 = regexp.MustCompile(`(?m)^[=-]+(\s|\n)|{{[A-Za-z]`)
asc_AGSScript_Matcher_0 = regexp.MustCompile(`(?m)^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])`)
ms_Groff_Matcher_0 = regexp.MustCompile(`(?mi)^[.'][a-z][a-z](\s|$)`)
mod_XML_Matcher_0 = regexp.MustCompile(`(?m)<!ENTITY `)
mod_ModulaDash2_Matcher_0 = regexp.MustCompile(`(?mi)^\s*MODULE [\w\.]+;`)
mod_ModulaDash2_Matcher_1 = regexp.MustCompile(`(?mi)^\s*END [\w\.]+;`)
pro_Prolog_Matcher_0 = regexp.MustCompile(`(?m)^[^#]+:-`)
pro_INI_Matcher_0 = regexp.MustCompile(`(?m)last_client=`)
pro_QMake_Matcher_0 = regexp.MustCompile(`(?m)HEADERS`)
pro_QMake_Matcher_1 = regexp.MustCompile(`(?m)SOURCES`)
pro_IDL_Matcher_0 = regexp.MustCompile(`(?m)^\s*function[ \w,]+$`)
asc_PublicKey_Matcher_0 = regexp.MustCompile(`(?m)^(----[- ]BEGIN|ssh-(rsa|dss)) `)
asc_AsciiDoc_Matcher_0 = regexp.MustCompile(`(?m)^[=-]+(\s|\n)|{{[A-Za-z]`)
asc_AGSScript_Matcher_0 = regexp.MustCompile(`(?m)^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])`)
f_Forth_Matcher_0 = regexp.MustCompile(`(?m)^: `)
f_FilebenchWML_Matcher_0 = regexp.MustCompile(`(?m)flowop`)
f_FORTRAN_Matcher_0 = regexp.MustCompile(`(?mi)^([c*][^abd-z]| (subroutine|program|end|data)\s|\s*!)`)
h_ObjectiveDashC_Matcher_0 = regexp.MustCompile(`(?m)^\s*(@(interface|class|protocol|property|end|synchronised|selector|implementation)\b|#import\s+.+\.h[">])`)
h_CPlusPlus_Matcher_0 = regexp.MustCompile(`(?m)^\s*#\s*include <(cstdint|string|vector|map|list|array|bitset|queue|stack|forward_list|unordered_map|unordered_set|(i|o|io)stream)>`)
h_CPlusPlus_Matcher_1 = regexp.MustCompile(`(?m)^\s*template\s*<`)
h_CPlusPlus_Matcher_2 = regexp.MustCompile(`(?m)^[ \t]*try`)
h_CPlusPlus_Matcher_3 = regexp.MustCompile(`(?m)^[ \t]*catch\s*\(`)
h_CPlusPlus_Matcher_4 = regexp.MustCompile(`(?m)^[ \t]*(class|(using[ \t]+)?namespace)\s+\w+`)
h_CPlusPlus_Matcher_5 = regexp.MustCompile(`(?m)^[ \t]*(private|public|protected):$`)
h_CPlusPlus_Matcher_6 = regexp.MustCompile(`(?m)std::\w+`)
lsp_CommonLisp_Matcher_0 = regexp.MustCompile(`(?mi)^\s*\((defun|in-package|defpackage) `)
lsp_NewLisp_Matcher_0 = regexp.MustCompile(`(?m)^\s*\(define `)
lisp_CommonLisp_Matcher_0 = regexp.MustCompile(`(?mi)^\s*\((defun|in-package|defpackage) `)
lisp_NewLisp_Matcher_0 = regexp.MustCompile(`(?m)^\s*\(define `)
md_Markdown_Matcher_0 = regexp.MustCompile(`(?mi)(^[-a-z0-9=#!\*\[|>])|<\/`)
md_Markdown_Matcher_1 = regexp.MustCompile(`(?m)^$`)
md_GCCmachinedescription_Matcher_0 = regexp.MustCompile(`(?m)^(;;|\(define_)`)
ms_Groff_Matcher_0 = regexp.MustCompile(`(?mi)^[.'][a-z][a-z](\s|$)`)
mod_XML_Matcher_0 = regexp.MustCompile(`(?m)<!ENTITY `)
mod_ModulaDash2_Matcher_0 = regexp.MustCompile(`(?mi)^\s*MODULE [\w\.]+;`)
mod_ModulaDash2_Matcher_1 = regexp.MustCompile(`(?mi)^\s*END [\w\.]+;`)
pro_Prolog_Matcher_0 = regexp.MustCompile(`(?m)^[^#]+:-`)
pro_INI_Matcher_0 = regexp.MustCompile(`(?m)last_client=`)
pro_QMake_Matcher_0 = regexp.MustCompile(`(?m)HEADERS`)
pro_QMake_Matcher_1 = regexp.MustCompile(`(?m)SOURCES`)
pro_IDL_Matcher_0 = regexp.MustCompile(`(?m)^\s*function[ \w,]+$`)
rpy_Python_Matcher_0 = regexp.MustCompile(`(?ms)(^(import|from|class|def)\s)`)
)

51
internal/code-generator/generator/test_files/content.test.go.tmpl
View File

@ -1,51 +0,0 @@
package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: {{ getCommit }}
import (
"path/filepath"
"regexp"
"strings"
)
func GetLanguageByContent(filename string, content []byte) (lang string, safe bool) {
ext := strings.ToLower(filepath.Ext(filename))
if fnMatcher, ok := matchers[ext]; ok {
lang, safe = fnMatcher(content)
return
}
return GetLanguageByExtension(filename)
}
type languageMatcher func ([]byte) (string, bool)
var matchers = map[string]languageMatcher{
{{ range $index, $disambiguator := . -}}
{{ printf "%q" $disambiguator.Extension }}: func(i []byte) (string, bool) {
{{ range $i, $language := $disambiguator.Languages -}}
{{- if not (avoidLanguage $language) }}
{{- if gt (len $language.Heuristics) 0 }}
{{- if gt $i 0 }} else {{ end -}}
if {{- range $j, $heuristic := $language.Heuristics }} {{ $heuristic.Name }}.Match(i)
{{- if lt $j (len $language.LogicRelations) }} {{index $language.LogicRelations $j}} {{- end -}} {{ end }} {
return {{ printf "%q" $language.Language }}, true
}
{{- end -}}
{{- end -}}
{{- end}}
return {{ returnLanguage $disambiguator.Languages }}, {{ safeLanguage $disambiguator.Languages }}
},
{{ end -}}
}
var (
{{ range $index, $heuristic := getAllHeuristics . -}}
{{ $heuristic.Name }} = regexp.MustCompile(`{{ $heuristic.Regexp }}`)
{{ end -}}
)

13
internal/code-generator/generator/test_files/documentation.test.go.tmpl
View File

@ -1,13 +0,0 @@
package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: {{ getCommit }}
import "gopkg.in/toqueteos/substring.v1"
var documentationMatchers = substring.Or(
{{range $regexp := . -}}
substring.Regexp(`{{ $regexp }}`),
{{end -}}
)

0
internal/code-generator/generator/test_files/languages.gold → internal/code-generator/generator/test_files/extensions.gold
View File

0
internal/code-generator/generator/test_files/languages.test.yml → internal/code-generator/generator/test_files/extensions.test.yml
View File

12
internal/code-generator/generator/test_files/filenames.gold Normal file
View File

@ -0,0 +1,12 @@
package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7
var languagesByFilename = map[string]string{
"APKBUILD": "Alpine Abuild",
"CMakeLists.txt": "CMake",
"Cakefile": "CoffeeScript",
"mix.lock": "Elixir",
}

13
internal/code-generator/generator/test_files/filenames.test.yml Normal file
View File

@ -0,0 +1,13 @@
---
Alpine Abuild:
filenames:
- APKBUILD
CMake:
filenames:
- CMakeLists.txt
CoffeeScript:
filenames:
- Cakefile
Elixir:
filenames:
- mix.lock

47
internal/code-generator/generator/test_files/heuristics.test.rb
View File

@ -11,6 +11,45 @@
end
end
fortran_rx = /^([c*][^abd-z]| (subroutine|program|end|data)\s|\s*!)/i
disambiguate ".f" do |data|
if /^: /.match(data)
Language["Forth"]
elsif data.include?("flowop")
Language["Filebench WML"]
elsif fortran_rx.match(data)
Language["FORTRAN"]
end
end
disambiguate ".h" do |data|
if ObjectiveCRegex.match(data)
Language["Objective-C"]
elsif (/^\s*#\s*include <(cstdint|string|vector|map|list|array|bitset|queue|stack|forward_list|unordered_map|unordered_set|(i|o|io)stream)>/.match(data) ||
/^\s*template\s*</.match(data) || /^[ \t]*try/.match(data) || /^[ \t]*catch\s*\(/.match(data) || /^[ \t]*(class|(using[ \t]+)?namespace)\s+\w+/.match(data) || /^[ \t]*(private|public|protected):$/.match(data) || /std::\w+/.match(data))
Language["C++"]
end
end
disambiguate ".lsp", ".lisp" do |data|
if /^\s*\((defun|in-package|defpackage) /i.match(data)
Language["Common Lisp"]
elsif /^\s*\(define /.match(data)
Language["NewLisp"]
end
end
disambiguate ".md" do |data|
if /(^[-a-z0-9=#!\*\[|>])|<\//i.match(data) || data.empty?
Language["Markdown"]
elsif /^(;;|\(define_)/.match(data)
Language["GCC machine description"]
else
Language["Markdown"]
end
end
disambiguate ".ms" do |data|
if /^[.'][a-z][a-z](\s|$)/i.match(data)
Language["Groff"]
@ -42,3 +81,11 @@
Language["IDL"]
end
end
disambiguate ".rpy" do |data|
if /(^(import|from|class|def)\s)/m.match(data)
Language["Python"]
else
Language["Ren'Py"]
end
end

11
internal/code-generator/generator/test_files/interpreters.test.go.tmpl
View File

@ -1,11 +0,0 @@
package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: {{ getCommit }}
var languagesByInterpreter = map[string][]string{
{{range $interpreter, $languages := . -}}
"{{ $interpreter }}": { {{- $languages | formatStringSlice -}} },
{{end -}}
}

11
internal/code-generator/generator/test_files/languages.test.go.tmpl
View File

@ -1,11 +0,0 @@
package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: {{ getCommit }}
var languagesByExtension = map[string][]string{
{{range $extension, $languages := . -}}
"{{ $extension }}": { {{- $languages | formatStringSlice -}} },
{{end -}}
}

20
internal/code-generator/main.go
View File

@ -8,11 +8,13 @@ import (
)
const (
// languages.go generation
languagesYAML = ".linguist/lib/linguist/languages.yml"
langFile = "languages.go"
languagesTmplPath = "internal/code-generator/assets/languages.go.tmpl"
languagesTmpl = "languages.go.tmpl"
// languages info file
languagesYAML = ".linguist/lib/linguist/languages.yml"
// extensions_map.go generation
extensionsFile = "extensions_map.go"
extensionsTmplPath = "internal/code-generator/assets/extensions.go.tmpl"
extensionsTmpl = "extensions.go.tmpl"
// content.go generation
heuristicsRuby = ".linguist/lib/linguist/heuristics.rb"
@ -42,6 +44,11 @@ const (
interpretersTmplPath = "internal/code-generator/assets/interpreters.go.tmpl"
interpretersTmpl = "interpreters.go.tmpl"
// filenames_map.go generation
filenamesFile = "filenames_map.go"
filenamesTmplPath = "internal/code-generator/assets/filenames.go.tmpl"
filenamesTmpl = "filenames.go.tmpl"
commitPath = ".git/refs/heads/master"
)
@ -61,12 +68,13 @@ func main() {
}
argsList := []*generatorArgs{
&generatorArgs{languagesYAML, langFile, languagesTmplPath, languagesTmpl, commit, generator.Languages},
&generatorArgs{languagesYAML, extensionsFile, extensionsTmplPath, extensionsTmpl, commit, generator.Extensions},
&generatorArgs{heuristicsRuby, contentFile, contentTmplPath, contentTmpl, commit, generator.Heuristics},
&generatorArgs{vendorYAML, vendorFile, vendorTmplPath, vendorTmpl, commit, generator.Vendor},
&generatorArgs{documentationYAML, documentationFile, documentationTmplPath, documentationTmpl, commit, generator.Documentation},
&generatorArgs{languagesYAML, typeFile, typeTmplPath, typeTmpl, commit, generator.Types},
&generatorArgs{languagesYAML, interpretersFile, interpretersTmplPath, interpretersTmpl, commit, generator.Interpreters},
&generatorArgs{languagesYAML, filenamesFile, filenamesTmplPath, filenamesTmpl, commit, generator.Filenames},
}
for _, args := range argsList {

4
shebang.go
View File

@ -7,6 +7,8 @@ import (
"strings"
)
const shebang = `#!`
var (
shebangExecHack = regexp.MustCompile(`exec (\w+).+\$0.+\$@`)
pythonVersion = regexp.MustCompile(`python\d\.\d+`)
@ -68,7 +70,7 @@ func getFirstLine(data []byte) []byte {
}
func hasShebang(line []byte) bool {
shebang := []byte{'#', '!'}
shebang := []byte(shebang)
return bytes.HasPrefix(line, shebang)
}