Added languages.go generator

cli/slinguist-generate/assets/languages.go.tmpl

@@ -0,0 +1,11 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: {{ getCommit }}
+
+var languagesByExtension = map[string][]string{
+	{{range $extension, $languages := . -}}
+	"{{ $extension }}":	{ {{- $languages | formatStringSlice -}} },
+	{{end -}}
+}

cli/slinguist-generate/generator/generator.go

@@ -0,0 +1,34 @@
+package generator
+
+import (
+	"go/format"
+	"io/ioutil"
+)
+
+// Func is the function's type that generate the files from templates.
+type Func func(dataToParse []byte, templatePath string, template string, commit string) ([]byte, error)
+
+// FromFile read data to parse from a file named fileToParse and write the generated source code to a file named outPath. The generated
+// source code is formated with gofmt and tagged with commit.
+func FromFile(fileToParse, outPath, tmplPath, tmplName, commit string, generate Func) error {
+	buf, err := ioutil.ReadFile(fileToParse)
+	if err != nil {
+		return err
+	}
+
+	source, err := generate(buf, tmplPath, tmplName, commit)
+	if err != nil {
+		return err
+	}
+
+	formatedSource, err := format.Source(source)
+	if err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(outPath, formatedSource, 0666); err != nil {
+		return err
+	}
+
+	return nil
+}

cli/slinguist-generate/generator/generator_test.go

@@ -0,0 +1,98 @@
+package generator
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	// FromFile test
+	formatedLangGold    = "test_files/formated_languages.gold"
+	formatedContentGold = "test_files/formated_content.gold"
+
+	// Languages test
+	ymlTestFile           = "test_files/languages.test.yml"
+	langGold              = "test_files/languages.gold"
+	languagesTestTmplPath = "test_files/languages.test.tmpl"
+	languagesTestTmplName = "languages.test.tmpl"
+	commitLangTest        = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7"
+)
+
+func TestFromFile(t *testing.T) {
+	goldLang, err := ioutil.ReadFile(formatedLangGold)
+	assert.NoError(t, err)
+
+	outPathLang, err := ioutil.TempFile("/tmp", "generator-test-")
+	assert.NoError(t, err)
+	defer os.Remove(outPathLang.Name())
+
+	tests := []struct {
+		name        string
+		fileToParse string
+		outPath     string
+		tmplPath    string
+		tmplName    string
+		commit      string
+		generate    Func
+		wantOut     []byte
+	}{
+		{
+			name:        "TestFromFile_Language",
+			fileToParse: ymlTestFile,
+			outPath:     outPathLang.Name(),
+			tmplPath:    languagesTestTmplPath,
+			tmplName:    languagesTestTmplName,
+			commit:      commitLangTest,
+			generate:    Languages,
+			wantOut:     goldLang,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := FromFile(tt.fileToParse, tt.outPath, tt.tmplPath, tt.tmplName, tt.commit, tt.generate)
+			assert.NoError(t, err)
+			out, err := ioutil.ReadFile(tt.outPath)
+			assert.NoError(t, err)
+			assert.EqualValues(t, tt.wantOut, out, fmt.Sprintf("FromFile() = %v, want %v", string(out), string(tt.wantOut)))
+		})
+	}
+}
+
+func TestLanguages(t *testing.T) {
+	gold, err := ioutil.ReadFile(langGold)
+	assert.NoError(t, err)
+
+	input, err := ioutil.ReadFile(ymlTestFile)
+	assert.NoError(t, err)
+
+	tests := []struct {
+		name     string
+		input    []byte
+		tmplPath string
+		tmplName string
+		commit   string
+		wantOut  []byte
+	}{
+		{
+			name:     "TestLanguages",
+			input:    input,
+			tmplPath: languagesTestTmplPath,
+			tmplName: languagesTestTmplName,
+			commit:   commitLangTest,
+			wantOut:  gold,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			out, err := Languages(tt.input, tt.tmplPath, tt.tmplName, tt.commit)
+			assert.NoError(t, err)
+			assert.EqualValues(t, tt.wantOut, out, fmt.Sprintf("Languages() = %v, want %v", string(out), string(tt.wantOut)))
+		})
+	}
+}

cli/slinguist-generate/generator/langs.go

@@ -0,0 +1,92 @@
+package generator
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"strings"
+	"text/template"
+
+	"gopkg.in/yaml.v2"
+)
+
+var (
+	// ErrExtensionsNotFound is the error returned if data parsed doesn't contain extensions.
+	ErrExtensionsNotFound = errors.New("extensions not found")
+)
+
+// Languages reads from buf and builds languages.go file from languagesTmplPath.
+func Languages(data []byte, languagesTmplPath, languagesTmplName, commit string) ([]byte, error) {
+	var yamlSlice yaml.MapSlice
+	if err := yaml.Unmarshal(data, &yamlSlice); err != nil {
+		return nil, err
+	}
+
+	languagesByExtension, err := buildExtensionLanguageMap(yamlSlice)
+	if err != nil {
+		return nil, err
+	}
+
+	buf := &bytes.Buffer{}
+	if err := executeLanguagesTemplate(buf, languagesByExtension, languagesTmplPath, languagesTmplName, commit); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+func buildExtensionLanguageMap(yamlSlice yaml.MapSlice) (map[string][]string, error) {
+	extensionLangsMap := make(map[string][]string)
+	for _, lang := range yamlSlice {
+		extensions, err := findExtensions(lang.Value.(yaml.MapSlice))
+		if err != nil && err != ErrExtensionsNotFound {
+			return nil, err
+		}
+
+		fillMap(extensionLangsMap, lang.Key.(string), extensions)
+	}
+
+	return extensionLangsMap, nil
+}
+
+func findExtensions(items yaml.MapSlice) ([]string, error) {
+	const extField = "extensions"
+	for _, item := range items {
+		if item.Key == extField {
+			extensions := toStringSlice(item.Value.([]interface{}))
+			return extensions, nil
+		}
+	}
+
+	return nil, ErrExtensionsNotFound
+}
+
+func toStringSlice(slice []interface{}) []string {
+	extensions := make([]string, 0, len(slice))
+	for _, element := range slice {
+		extension := element.(string)
+		extensions = append(extensions, extension)
+	}
+
+	return extensions
+}
+
+func fillMap(extensionLangs map[string][]string, lang string, extensions []string) {
+	for _, extension := range extensions {
+		extensionLangs[extension] = append(extensionLangs[extension], lang)
+	}
+}
+
+func executeLanguagesTemplate(out io.Writer, languagesByExtension map[string][]string, languagesTmplPath, languagesTmpl, commit string) error {
+	fmap := template.FuncMap{
+		"getCommit":         func() string { return commit },
+		"formatStringSlice": func(slice []string) string { return `"` + strings.Join(slice, `","`) + `"` },
+	}
+
+	t := template.Must(template.New(languagesTmpl).Funcs(fmap).ParseFiles(languagesTmplPath))
+	if err := t.Execute(out, languagesByExtension); err != nil {
+		return err
+	}
+
+	return nil
+}

cli/slinguist-generate/generator/test_files/formated_languages.gold

@@ -0,0 +1,12 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7
+
+var languagesByExtension = map[string][]string{
+	".abap": {"ABAP"},
+	".abnf": {"ABNF"},
+	".bsl":  {"1C Enterprise"},
+	".os":   {"1C Enterprise"},
+}

cli/slinguist-generate/generator/test_files/languages.gold

@@ -0,0 +1,12 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7
+
+var languagesByExtension = map[string][]string{
+	".abap":	{"ABAP"},
+	".abnf":	{"ABNF"},
+	".bsl":	{"1C Enterprise"},
+	".os":	{"1C Enterprise"},
+	}

cli/slinguist-generate/generator/test_files/languages.test.go.tmpl

@@ -0,0 +1,11 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: {{ getCommit }}
+
+var languagesByExtension = map[string][]string{
+	{{range $extension, $languages := . -}}
+	"{{ $extension }}":	{ {{- $languages | formatStringSlice -}} },
+	{{end -}}
+}

cli/slinguist-generate/generator/test_files/languages.test.tmpl

@@ -0,0 +1,11 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: {{ getCommit }}
+
+var languagesByExtension = map[string][]string{
+	{{range $extension, $languages := . -}}
+	"{{ $extension }}":	{ {{- $languages | formatStringSlice -}} },
+	{{end -}}
+}

cli/slinguist-generate/generator/test_files/languages.test.yml

@@ -0,0 +1,24 @@
+---
+1C Enterprise:
+  type: programming
+  color: "#814CCC"
+  extensions:
+  - ".bsl"
+  - ".os"
+  tm_scope: source.bsl
+  ace_mode: text
+  language_id: 0
+ABAP:
+  type: programming
+  color: "#E8274B"
+  extensions:
+  - ".abap"
+  ace_mode: abap
+  language_id: 1
+ABNF:
+  type: data
+  ace_mode: text
+  extensions:
+  - ".abnf"
+  tm_scope: source.abnf
+  language_id: 429

cli/slinguist-generate/main.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"io/ioutil"
+	"log"
+
+	"srcd.works/simple-linguist.v1/cli/slinguist-generate/generator"
+)
+
+const (
+	languagesYAML     = ".linguist/lib/linguist/languages.yml"
+	langFile          = "languages.go"
+	languagesTmplPath = "cli/slinguist-generate/assets/languages.go.tmpl"
+	languagesTmpl     = "languages.go.tmpl"
+
+	heuristicsRuby  = ".linguist/lib/linguist/heuristics.rb"
+	contentFile     = "content.go"
+	contentTmplPath = "cli/slinguist-generate/assets/content.go.tmpl"
+	contentTmpl     = "content.go.tmpl"
+
+	commitPath = ".git/refs/heads/master"
+)
+
+func main() {
+	commit, err := getCommit(commitPath)
+	if err != nil {
+		log.Printf("couldn't find commit: %v", err)
+	}
+
+	if err := generator.FromFile(languagesYAML, langFile, languagesTmplPath, languagesTmpl, commit, generator.Languages); err != nil {
+		log.Println(err)
+	}
+}
+
+func getCommit(path string) (string, error) {
+	commit, err := ioutil.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+
+	return string(commit), nil
+}

extension_test.go

@@ -12,7 +12,7 @@ func (s *TSuite) TestGetLanguageByExtension(c *C) {
 	c.Assert(safe, Equals, true)
 
 	lang, safe = GetLanguageByExtension("foo.go.php")
-	c.Assert(lang, Equals, "PHP")
+	c.Assert(lang, Equals, "Hack")
 	c.Assert(safe, Equals, false)
 }
 

generate.go

@@ -0,0 +1,3 @@
+package slinguist
+
+//go:generate go run cli/slinguist-generate/main.go