mirror of
https://github.com/ralsina/tartrazine.git
synced 2025-06-19 14:43:05 -03:00
move content.go generation to internal
This commit is contained in:
Manuel Carmona
488
internal/code-generator/generator/heuristics.go
Normal file
488
internal/code-generator/generator/heuristics.go
Normal file
@ -0,0 +1,488 @@
|
||||
package generator
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"text/template"
|
||||
)
|
||||
|
||||
// Heuristics reads from buf and builds content.go file from contentTmplPath.
|
||||
func Heuristics(heuristics []byte, contentTmplPath, contentTmplName, commit string) ([]byte, error) {
|
||||
disambiguators, err := getDisambiguators(heuristics)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
if err := executeContentTemplate(buf, disambiguators, contentTmplPath, contentTmplName, commit); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
const unknownLanguage = "OtherLanguage"
|
||||
|
||||
var (
|
||||
disambLine = regexp.MustCompile(`^(\s*)disambiguate`)
|
||||
definedRegs = make(map[string]string)
|
||||
illegalCharacter = map[string]string{
|
||||
"#": "Sharp",
|
||||
"+": "Plus",
|
||||
"-": "Dash",
|
||||
}
|
||||
)
|
||||
|
||||
type disambiguator struct {
|
||||
Extension string `json:"extension,omitempty"`
|
||||
Languages []*languageHeuristics `json:"languages,omitempty"`
|
||||
}
|
||||
|
||||
func (d *disambiguator) setHeuristicsNames() {
|
||||
for _, lang := range d.Languages {
|
||||
for i, heuristic := range lang.Heuristics {
|
||||
name := buildName(d.Extension, lang.Language, i)
|
||||
heuristic.Name = name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func buildName(extension, language string, id int) string {
|
||||
extension = strings.TrimPrefix(extension, `.`)
|
||||
language = strings.Join(strings.Fields(language), ``)
|
||||
name := strings.Join([]string{extension, language, "Matcher", strconv.Itoa(id)}, `_`)
|
||||
for k, v := range illegalCharacter {
|
||||
if strings.Contains(name, k) {
|
||||
name = strings.Replace(name, k, v, -1)
|
||||
}
|
||||
}
|
||||
|
||||
return name
|
||||
}
|
||||
|
||||
type languageHeuristics struct {
|
||||
Language string `json:"language,omitempty"`
|
||||
Heuristics []*heuristic `json:"heuristics,omitempty"`
|
||||
LogicRelations []string `json:"logic_relations,omitempty"`
|
||||
}
|
||||
|
||||
func (l *languageHeuristics) clone() (*languageHeuristics, error) {
|
||||
language := l.Language
|
||||
logicRels := make([]string, len(l.LogicRelations))
|
||||
if copy(logicRels, l.LogicRelations) != len(l.LogicRelations) {
|
||||
return nil, fmt.Errorf("error copying logic relations")
|
||||
}
|
||||
|
||||
heuristics := make([]*heuristic, 0, len(l.Heuristics))
|
||||
for _, h := range l.Heuristics {
|
||||
heuristic := *h
|
||||
heuristics = append(heuristics, &heuristic)
|
||||
}
|
||||
|
||||
clone := &languageHeuristics{
|
||||
Language: language,
|
||||
Heuristics: heuristics,
|
||||
LogicRelations: logicRels,
|
||||
}
|
||||
|
||||
return clone, nil
|
||||
}
|
||||
|
||||
type heuristic struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Regexp string `json:"regexp,omitempty"`
|
||||
}
|
||||
|
||||
// A disambiguate block looks like:
|
||||
// disambiguate ".mod", ".extension" do |data|
|
||||
// if data.include?('<!ENTITY ') && data.include?('patata')
|
||||
// Language["XML"]
|
||||
// elsif /^\s*MODULE [\w\.]+;/i.match(data) || /^\s*END [\w\.]+;/i.match(data) || data.empty?
|
||||
// Language["Modula-2"]
|
||||
// elsif (/^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data))
|
||||
// Language["Scala"]
|
||||
// elsif (data.include?("gap> "))
|
||||
// Language["GAP"]
|
||||
// else
|
||||
// [Language["Linux Kernel Module"], Language["AMPL"]]
|
||||
// end
|
||||
// end
|
||||
func getDisambiguators(heuristics []byte) ([]*disambiguator, error) {
|
||||
seenExtensions := map[string]bool{}
|
||||
buf := bufio.NewScanner(bytes.NewReader(heuristics))
|
||||
disambiguators := make([]*disambiguator, 0, 50)
|
||||
for buf.Scan() {
|
||||
line := buf.Text()
|
||||
if disambLine.MatchString(line) {
|
||||
d, err := parseDisambiguators(line, buf, seenExtensions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
disambiguators = append(disambiguators, d...)
|
||||
}
|
||||
|
||||
lookForRegexpVariables(line)
|
||||
}
|
||||
|
||||
if err := buf.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return disambiguators, nil
|
||||
}
|
||||
|
||||
func lookForRegexpVariables(line string) {
|
||||
if strings.Contains(line, "ObjectiveCRegex = ") {
|
||||
line = strings.TrimSpace(line)
|
||||
reg := strings.TrimPrefix(line, "ObjectiveCRegex = ")
|
||||
definedRegs["ObjectiveCRegex"] = reg
|
||||
}
|
||||
|
||||
if strings.Contains(line, "fortran_rx = ") {
|
||||
line = strings.TrimSpace(line)
|
||||
reg := strings.TrimPrefix(line, "fortran_rx = ")
|
||||
definedRegs["fortran_rx"] = reg
|
||||
}
|
||||
}
|
||||
|
||||
func parseDisambiguators(line string, buf *bufio.Scanner, seenExtensions map[string]bool) ([]*disambiguator, error) {
|
||||
disambList := make([]*disambiguator, 0, 2)
|
||||
splitted := strings.Fields(line)
|
||||
|
||||
for _, v := range splitted {
|
||||
if strings.HasPrefix(v, `"`) {
|
||||
extension := strings.Trim(v, `",`)
|
||||
if _, ok := seenExtensions[extension]; !ok {
|
||||
d := &disambiguator{Extension: extension}
|
||||
disambList = append(disambList, d)
|
||||
seenExtensions[extension] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
langsHeuristics, err := getLanguagesHeuristics(buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i, disamb := range disambList {
|
||||
lh := langsHeuristics
|
||||
if i != 0 {
|
||||
lh = cloneLanguagesHeuristics(langsHeuristics)
|
||||
}
|
||||
|
||||
disamb.Languages = lh
|
||||
disamb.setHeuristicsNames()
|
||||
}
|
||||
|
||||
return disambList, nil
|
||||
}
|
||||
|
||||
func cloneLanguagesHeuristics(list []*languageHeuristics) []*languageHeuristics {
|
||||
cloneList := make([]*languageHeuristics, 0, len(list))
|
||||
for _, langHeu := range list {
|
||||
clone, _ := langHeu.clone()
|
||||
cloneList = append(cloneList, clone)
|
||||
}
|
||||
|
||||
return cloneList
|
||||
}
|
||||
|
||||
func getLanguagesHeuristics(buf *bufio.Scanner) ([]*languageHeuristics, error) {
|
||||
langsList := make([][]string, 0, 2)
|
||||
heuristicsList := make([][]*heuristic, 0, 1)
|
||||
logicRelsList := make([][]string, 0, 1)
|
||||
|
||||
lastWasMatch := false
|
||||
for buf.Scan() {
|
||||
line := buf.Text()
|
||||
if strings.TrimSpace(line) == "end" {
|
||||
break
|
||||
}
|
||||
|
||||
if hasRegExp(line) {
|
||||
line := cleanRegExpLine(line)
|
||||
|
||||
logicRels := getLogicRelations(line)
|
||||
heuristics := getHeuristics(line)
|
||||
if lastWasMatch {
|
||||
i := len(heuristicsList) - 1
|
||||
heuristicsList[i] = append(heuristicsList[i], heuristics...)
|
||||
i = len(logicRelsList) - 1
|
||||
logicRelsList[i] = append(logicRelsList[i], logicRels...)
|
||||
} else {
|
||||
heuristicsList = append(heuristicsList, heuristics)
|
||||
logicRelsList = append(logicRelsList, logicRels)
|
||||
}
|
||||
|
||||
lastWasMatch = true
|
||||
}
|
||||
|
||||
if strings.Contains(line, "Language") {
|
||||
langs := getLanguages(line)
|
||||
langsList = append(langsList, langs)
|
||||
lastWasMatch = false
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if err := buf.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
langsHeuristics := buildLanguagesHeuristics(langsList, heuristicsList, logicRelsList)
|
||||
return langsHeuristics, nil
|
||||
}
|
||||
|
||||
func hasRegExp(line string) bool {
|
||||
return strings.Contains(line, ".match") || strings.Contains(line, ".include?") || strings.Contains(line, ".empty?")
|
||||
}
|
||||
|
||||
func cleanRegExpLine(line string) string {
|
||||
if strings.Contains(line, "if ") {
|
||||
line = line[strings.Index(line, `if `)+3:]
|
||||
}
|
||||
|
||||
line = strings.TrimSpace(line)
|
||||
line = strings.TrimPrefix(line, `(`)
|
||||
if strings.Contains(line, "))") {
|
||||
line = strings.TrimSuffix(line, `)`)
|
||||
}
|
||||
|
||||
return line
|
||||
}
|
||||
|
||||
func getLogicRelations(line string) []string {
|
||||
rels := make([]string, 0)
|
||||
splitted := strings.Split(line, "||")
|
||||
for i, v := range splitted {
|
||||
if strings.Contains(v, "&&") {
|
||||
rels = append(rels, "&&")
|
||||
}
|
||||
|
||||
if i < len(splitted)-1 {
|
||||
rels = append(rels, "||")
|
||||
}
|
||||
}
|
||||
|
||||
if len(rels) == 0 {
|
||||
rels = nil
|
||||
}
|
||||
|
||||
return rels
|
||||
}
|
||||
|
||||
func getHeuristics(line string) []*heuristic {
|
||||
splitted := splitByLogicOps(line)
|
||||
heuristics := make([]*heuristic, 0, len(splitted))
|
||||
for _, v := range splitted {
|
||||
v = strings.TrimSpace(v)
|
||||
var reg string
|
||||
|
||||
if strings.Contains(v, ".match") {
|
||||
reg = v[:strings.Index(v, ".match")]
|
||||
reg = replaceRegexpVariables(reg)
|
||||
}
|
||||
|
||||
if strings.Contains(v, ".include?") {
|
||||
reg = includeToRegExp(v)
|
||||
}
|
||||
|
||||
if strings.Contains(v, ".empty?") {
|
||||
reg = `^$`
|
||||
}
|
||||
|
||||
if reg != "" {
|
||||
reg = convToValidRegexp(reg)
|
||||
heuristics = append(heuristics, &heuristic{Regexp: reg})
|
||||
}
|
||||
}
|
||||
|
||||
return heuristics
|
||||
}
|
||||
|
||||
func splitByLogicOps(line string) []string {
|
||||
splitted := make([]string, 0, 1)
|
||||
splitOr := strings.Split(line, "||")
|
||||
for _, v := range splitOr {
|
||||
splitAnd := strings.Split(v, "&&")
|
||||
splitted = append(splitted, splitAnd...)
|
||||
}
|
||||
|
||||
return splitted
|
||||
}
|
||||
|
||||
func replaceRegexpVariables(reg string) string {
|
||||
repl := reg
|
||||
if v, ok := definedRegs[reg]; ok {
|
||||
repl = v
|
||||
}
|
||||
|
||||
return repl
|
||||
}
|
||||
|
||||
func convToValidRegexp(reg string) string {
|
||||
// example: `/^(\s*)(<Project|<Import|<Property|<?xml|xmlns)/i``
|
||||
// Ruby modifier "m" matches multiple lines, recognizing newlines as normal characters, Go use flag "s" for that.
|
||||
const (
|
||||
caseSensitive = "i"
|
||||
matchEOL = "s"
|
||||
|
||||
rubyCaseSensitive = "i"
|
||||
rubyMultiLine = "m"
|
||||
)
|
||||
|
||||
reg = strings.TrimPrefix(reg, `/`)
|
||||
flags := "(?m"
|
||||
lastSlash := strings.LastIndex(reg, `/`)
|
||||
if lastSlash == -1 {
|
||||
return flags + ")" + reg
|
||||
}
|
||||
|
||||
specialChars := reg[lastSlash:]
|
||||
reg = reg[:lastSlash]
|
||||
if lastSlash == len(reg)-1 {
|
||||
return flags + ")" + reg
|
||||
}
|
||||
|
||||
if strings.Contains(specialChars, rubyCaseSensitive) {
|
||||
flags = flags + caseSensitive
|
||||
}
|
||||
|
||||
if strings.Contains(specialChars, rubyMultiLine) {
|
||||
flags = flags + matchEOL
|
||||
}
|
||||
|
||||
return flags + ")" + reg
|
||||
}
|
||||
|
||||
func includeToRegExp(include string) string {
|
||||
content := include[strings.Index(include, `(`)+1 : strings.Index(include, `)`)]
|
||||
content = strings.Trim(content, `"'`)
|
||||
return regexp.QuoteMeta(content)
|
||||
}
|
||||
|
||||
func getLanguages(line string) []string {
|
||||
languages := make([]string, 0)
|
||||
splitted := strings.Split(line, `,`)
|
||||
for _, lang := range splitted {
|
||||
lang = trimLanguage(lang)
|
||||
languages = append(languages, lang)
|
||||
}
|
||||
|
||||
return languages
|
||||
}
|
||||
|
||||
func trimLanguage(enclosedLang string) string {
|
||||
lang := strings.TrimSpace(enclosedLang)
|
||||
lang = lang[strings.Index(lang, `"`)+1:]
|
||||
lang = lang[:strings.Index(lang, `"`)]
|
||||
return lang
|
||||
}
|
||||
|
||||
func buildLanguagesHeuristics(langsList [][]string, heuristicsList [][]*heuristic, logicRelsList [][]string) []*languageHeuristics {
|
||||
langsHeuristics := make([]*languageHeuristics, 0, len(langsList))
|
||||
for i, langSlice := range langsList {
|
||||
var heuristics []*heuristic
|
||||
if i < len(heuristicsList) {
|
||||
heuristics = heuristicsList[i]
|
||||
}
|
||||
|
||||
var rels []string
|
||||
if i < len(logicRelsList) {
|
||||
rels = logicRelsList[i]
|
||||
}
|
||||
|
||||
for _, lang := range langSlice {
|
||||
lh := &languageHeuristics{
|
||||
Language: lang,
|
||||
Heuristics: heuristics,
|
||||
LogicRelations: rels,
|
||||
}
|
||||
|
||||
langsHeuristics = append(langsHeuristics, lh)
|
||||
}
|
||||
}
|
||||
|
||||
return langsHeuristics
|
||||
}
|
||||
|
||||
func executeContentTemplate(out io.Writer, disambiguators []*disambiguator, contentTmplPath, contentTmpl, commit string) error {
|
||||
fmap := template.FuncMap{
|
||||
"getCommit": func() string { return commit },
|
||||
"getAllHeuristics": getAllHeuristics,
|
||||
"returnLanguage": returnLanguage,
|
||||
"safeLanguage": safeLanguage,
|
||||
"avoidLanguage": avoidLanguage,
|
||||
}
|
||||
|
||||
t := template.Must(template.New(contentTmpl).Funcs(fmap).ParseFiles(contentTmplPath))
|
||||
if err := t.Execute(out, disambiguators); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getAllHeuristics(disambiguators []*disambiguator) []*heuristic {
|
||||
heuristics := make([]*heuristic, 0)
|
||||
for _, disamb := range disambiguators {
|
||||
for _, lang := range disamb.Languages {
|
||||
if !avoidLanguage(lang) {
|
||||
heuristics = append(heuristics, lang.Heuristics...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return heuristics
|
||||
}
|
||||
|
||||
func avoidLanguage(lang *languageHeuristics) bool {
|
||||
// necessary to avoid corner cases
|
||||
for _, heuristic := range lang.Heuristics {
|
||||
if containsInvalidRegexp(heuristic.Regexp) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func containsInvalidRegexp(reg string) bool {
|
||||
return strings.Contains(reg, `(?<`) || strings.Contains(reg, `\1`)
|
||||
}
|
||||
|
||||
func returnLanguage(langsHeuristics []*languageHeuristics) string {
|
||||
lang, _ := returnLangAndSafe(langsHeuristics)
|
||||
return lang
|
||||
}
|
||||
|
||||
func safeLanguage(langsHeuristics []*languageHeuristics) bool {
|
||||
_, safe := returnLangAndSafe(langsHeuristics)
|
||||
return safe
|
||||
}
|
||||
|
||||
func returnLangAndSafe(langsHeuristics []*languageHeuristics) (string, bool) {
|
||||
// at the moment, only returns one string although might be exists several language to return as a []string.
|
||||
langs := make([]string, 0)
|
||||
for _, langHeu := range langsHeuristics {
|
||||
if len(langHeu.Heuristics) == 0 {
|
||||
langs = append(langs, `"`+langHeu.Language+`"`)
|
||||
}
|
||||
}
|
||||
|
||||
lang := unknownLanguage
|
||||
safe := false
|
||||
if len(langs) != 0 {
|
||||
lang = langs[0]
|
||||
safe = len(langs) == 1
|
||||
}
|
||||
|
||||
return lang, safe
|
||||
}
|
||||
81
internal/code-generator/generator/test_files/content.gold
Normal file
81
internal/code-generator/generator/test_files/content.gold
Normal file
@ -0,0 +1,81 @@
|
||||
package slinguist
|
||||
|
||||
// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
|
||||
// THIS FILE SHOULD NOT BE EDITED BY HAND
|
||||
// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func GetLanguageByContent(filename string, content []byte) (lang string, safe bool) {
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
if fnMatcher, ok := matchers[ext]; ok {
|
||||
lang, safe = fnMatcher(content)
|
||||
return
|
||||
}
|
||||
|
||||
return GetLanguageByExtension(filename)
|
||||
}
|
||||
|
||||
type languageMatcher func ([]byte) (string, bool)
|
||||
|
||||
var matchers = map[string]languageMatcher{
|
||||
".asc": func(i []byte) (string, bool) {
|
||||
if asc_PublicKey_Matcher_0.Match(i) {
|
||||
return "Public Key", true
|
||||
} else if asc_AsciiDoc_Matcher_0.Match(i) {
|
||||
return "AsciiDoc", true
|
||||
} else if asc_AGSScript_Matcher_0.Match(i) {
|
||||
return "AGS Script", true
|
||||
}
|
||||
|
||||
return OtherLanguage, false
|
||||
},
|
||||
".ms": func(i []byte) (string, bool) {
|
||||
if ms_Groff_Matcher_0.Match(i) {
|
||||
return "Groff", true
|
||||
}
|
||||
|
||||
return "MAXScript", true
|
||||
},
|
||||
".mod": func(i []byte) (string, bool) {
|
||||
if mod_XML_Matcher_0.Match(i) {
|
||||
return "XML", true
|
||||
} else if mod_ModulaDash2_Matcher_0.Match(i) || mod_ModulaDash2_Matcher_1.Match(i) {
|
||||
return "Modula-2", true
|
||||
}
|
||||
|
||||
return "Linux Kernel Module", false
|
||||
},
|
||||
".pro": func(i []byte) (string, bool) {
|
||||
if pro_Prolog_Matcher_0.Match(i) {
|
||||
return "Prolog", true
|
||||
} else if pro_INI_Matcher_0.Match(i) {
|
||||
return "INI", true
|
||||
} else if pro_QMake_Matcher_0.Match(i) && pro_QMake_Matcher_1.Match(i) {
|
||||
return "QMake", true
|
||||
} else if pro_IDL_Matcher_0.Match(i) {
|
||||
return "IDL", true
|
||||
}
|
||||
|
||||
return OtherLanguage, false
|
||||
},
|
||||
}
|
||||
|
||||
var (
|
||||
asc_PublicKey_Matcher_0 = regexp.MustCompile(`(?m)^(----[- ]BEGIN|ssh-(rsa|dss)) `)
|
||||
asc_AsciiDoc_Matcher_0 = regexp.MustCompile(`(?m)^[=-]+(\s|\n)|{{[A-Za-z]`)
|
||||
asc_AGSScript_Matcher_0 = regexp.MustCompile(`(?m)^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])`)
|
||||
ms_Groff_Matcher_0 = regexp.MustCompile(`(?mi)^[.'][a-z][a-z](\s|$)`)
|
||||
mod_XML_Matcher_0 = regexp.MustCompile(`(?m)<!ENTITY `)
|
||||
mod_ModulaDash2_Matcher_0 = regexp.MustCompile(`(?mi)^\s*MODULE [\w\.]+;`)
|
||||
mod_ModulaDash2_Matcher_1 = regexp.MustCompile(`(?mi)^\s*END [\w\.]+;`)
|
||||
pro_Prolog_Matcher_0 = regexp.MustCompile(`(?m)^[^#]+:-`)
|
||||
pro_INI_Matcher_0 = regexp.MustCompile(`(?m)last_client=`)
|
||||
pro_QMake_Matcher_0 = regexp.MustCompile(`(?m)HEADERS`)
|
||||
pro_QMake_Matcher_1 = regexp.MustCompile(`(?m)SOURCES`)
|
||||
pro_IDL_Matcher_0 = regexp.MustCompile(`(?m)^\s*function[ \w,]+$`)
|
||||
)
|
||||
@ -0,0 +1,51 @@
|
||||
package slinguist
|
||||
|
||||
// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
|
||||
// THIS FILE SHOULD NOT BE EDITED BY HAND
|
||||
// Extracted from github/linguist commit: {{ getCommit }}
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func GetLanguageByContent(filename string, content []byte) (lang string, safe bool) {
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
if fnMatcher, ok := matchers[ext]; ok {
|
||||
lang, safe = fnMatcher(content)
|
||||
return
|
||||
}
|
||||
|
||||
return GetLanguageByExtension(filename)
|
||||
}
|
||||
|
||||
type languageMatcher func ([]byte) (string, bool)
|
||||
|
||||
var matchers = map[string]languageMatcher{
|
||||
{{ range $index, $disambiguator := . -}}
|
||||
{{ printf "%q" $disambiguator.Extension }}: func(i []byte) (string, bool) {
|
||||
{{ range $i, $language := $disambiguator.Languages -}}
|
||||
|
||||
{{- if not (avoidLanguage $language) }}
|
||||
{{- if gt (len $language.Heuristics) 0 }}
|
||||
{{- if gt $i 0 }} else {{ end -}}
|
||||
if {{- range $j, $heuristic := $language.Heuristics }} {{ $heuristic.Name }}.Match(i)
|
||||
{{- if lt $j (len $language.LogicRelations) }} {{index $language.LogicRelations $j}} {{- end -}} {{ end }} {
|
||||
return {{ printf "%q" $language.Language }}, true
|
||||
}
|
||||
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end}}
|
||||
|
||||
return {{ returnLanguage $disambiguator.Languages }}, {{ safeLanguage $disambiguator.Languages }}
|
||||
},
|
||||
{{ end -}}
|
||||
}
|
||||
|
||||
var (
|
||||
{{ range $index, $heuristic := getAllHeuristics . -}}
|
||||
{{ $heuristic.Name }} = regexp.MustCompile(`{{ $heuristic.Regexp }}`)
|
||||
{{ end -}}
|
||||
)
|
||||
@ -0,0 +1,81 @@
|
||||
package slinguist
|
||||
|
||||
// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
|
||||
// THIS FILE SHOULD NOT BE EDITED BY HAND
|
||||
// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func GetLanguageByContent(filename string, content []byte) (lang string, safe bool) {
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
if fnMatcher, ok := matchers[ext]; ok {
|
||||
lang, safe = fnMatcher(content)
|
||||
return
|
||||
}
|
||||
|
||||
return GetLanguageByExtension(filename)
|
||||
}
|
||||
|
||||
type languageMatcher func([]byte) (string, bool)
|
||||
|
||||
var matchers = map[string]languageMatcher{
|
||||
".asc": func(i []byte) (string, bool) {
|
||||
if asc_PublicKey_Matcher_0.Match(i) {
|
||||
return "Public Key", true
|
||||
} else if asc_AsciiDoc_Matcher_0.Match(i) {
|
||||
return "AsciiDoc", true
|
||||
} else if asc_AGSScript_Matcher_0.Match(i) {
|
||||
return "AGS Script", true
|
||||
}
|
||||
|
||||
return OtherLanguage, false
|
||||
},
|
||||
".ms": func(i []byte) (string, bool) {
|
||||
if ms_Groff_Matcher_0.Match(i) {
|
||||
return "Groff", true
|
||||
}
|
||||
|
||||
return "MAXScript", true
|
||||
},
|
||||
".mod": func(i []byte) (string, bool) {
|
||||
if mod_XML_Matcher_0.Match(i) {
|
||||
return "XML", true
|
||||
} else if mod_ModulaDash2_Matcher_0.Match(i) || mod_ModulaDash2_Matcher_1.Match(i) {
|
||||
return "Modula-2", true
|
||||
}
|
||||
|
||||
return "Linux Kernel Module", false
|
||||
},
|
||||
".pro": func(i []byte) (string, bool) {
|
||||
if pro_Prolog_Matcher_0.Match(i) {
|
||||
return "Prolog", true
|
||||
} else if pro_INI_Matcher_0.Match(i) {
|
||||
return "INI", true
|
||||
} else if pro_QMake_Matcher_0.Match(i) && pro_QMake_Matcher_1.Match(i) {
|
||||
return "QMake", true
|
||||
} else if pro_IDL_Matcher_0.Match(i) {
|
||||
return "IDL", true
|
||||
}
|
||||
|
||||
return OtherLanguage, false
|
||||
},
|
||||
}
|
||||
|
||||
var (
|
||||
asc_PublicKey_Matcher_0 = regexp.MustCompile(`(?m)^(----[- ]BEGIN|ssh-(rsa|dss)) `)
|
||||
asc_AsciiDoc_Matcher_0 = regexp.MustCompile(`(?m)^[=-]+(\s|\n)|{{[A-Za-z]`)
|
||||
asc_AGSScript_Matcher_0 = regexp.MustCompile(`(?m)^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])`)
|
||||
ms_Groff_Matcher_0 = regexp.MustCompile(`(?mi)^[.'][a-z][a-z](\s|$)`)
|
||||
mod_XML_Matcher_0 = regexp.MustCompile(`(?m)<!ENTITY `)
|
||||
mod_ModulaDash2_Matcher_0 = regexp.MustCompile(`(?mi)^\s*MODULE [\w\.]+;`)
|
||||
mod_ModulaDash2_Matcher_1 = regexp.MustCompile(`(?mi)^\s*END [\w\.]+;`)
|
||||
pro_Prolog_Matcher_0 = regexp.MustCompile(`(?m)^[^#]+:-`)
|
||||
pro_INI_Matcher_0 = regexp.MustCompile(`(?m)last_client=`)
|
||||
pro_QMake_Matcher_0 = regexp.MustCompile(`(?m)HEADERS`)
|
||||
pro_QMake_Matcher_1 = regexp.MustCompile(`(?m)SOURCES`)
|
||||
pro_IDL_Matcher_0 = regexp.MustCompile(`(?m)^\s*function[ \w,]+$`)
|
||||
)
|
||||
@ -0,0 +1,44 @@
|
||||
# Common heuristics
|
||||
ObjectiveCRegex = /^\s*(@(interface|class|protocol|property|end|synchronised|selector|implementation)\b|#import\s+.+\.h[">])/
|
||||
|
||||
disambiguate ".asc" do |data|
|
||||
if /^(----[- ]BEGIN|ssh-(rsa|dss)) /.match(data)
|
||||
Language["Public Key"]
|
||||
elsif /^[=-]+(\s|\n)|{{[A-Za-z]/.match(data)
|
||||
Language["AsciiDoc"]
|
||||
elsif /^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])/.match(data)
|
||||
Language["AGS Script"]
|
||||
end
|
||||
end
|
||||
|
||||
disambiguate ".ms" do |data|
|
||||
if /^[.'][a-z][a-z](\s|$)/i.match(data)
|
||||
Language["Groff"]
|
||||
elsif /(?<!\S)\.(include|globa?l)\s/.match(data) || /(?<!\/\*)(\A|\n)\s*\.[A-Za-z]/.match(data.gsub(/"([^\\"]|\\.)*"|'([^\\']|\\.)*'|\\\s*(?:--.*)?\n/, ""))
|
||||
Language["GAS"]
|
||||
else
|
||||
Language["MAXScript"]
|
||||
end
|
||||
end
|
||||
|
||||
disambiguate ".mod" do |data|
|
||||
if data.include?('<!ENTITY ')
|
||||
Language["XML"]
|
||||
elsif /^\s*MODULE [\w\.]+;/i.match(data) || /^\s*END [\w\.]+;/i.match(data)
|
||||
Language["Modula-2"]
|
||||
else
|
||||
[Language["Linux Kernel Module"], Language["AMPL"]]
|
||||
end
|
||||
end
|
||||
|
||||
disambiguate ".pro" do |data|
|
||||
if /^[^#]+:-/.match(data)
|
||||
Language["Prolog"]
|
||||
elsif data.include?("last_client=")
|
||||
Language["INI"]
|
||||
elsif data.include?("HEADERS") && data.include?("SOURCES")
|
||||
Language["QMake"]
|
||||
elsif /^\s*function[ \w,]+$/.match(data)
|
||||
Language["IDL"]
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user