tartrazine/internal/code-generator/generator/heuristics.go
2017-06-15 10:07:22 +02:00

490 lines
12 KiB
Go

package generator
import (
"bufio"
"bytes"
"fmt"
"io"
"io/ioutil"
"regexp"
"strconv"
"strings"
"text/template"
)
// Heuristics reads from fileToParse and builds source file from tmplPath. It's comply with type File signature.
func Heuristics(fileToParse, samplesDir, outPath, tmplPath, tmplName, commit string) error {
data, err := ioutil.ReadFile(fileToParse)
if err != nil {
return err
}
disambiguators, err := getDisambiguators(data)
if err != nil {
return err
}
buf := &bytes.Buffer{}
if err := executeContentTemplate(buf, disambiguators, tmplPath, tmplName, commit); err != nil {
return err
}
return formatedWrite(outPath, buf.Bytes())
}
const (
unknownLanguage = "OtherLanguage"
emptyFile = "^$"
)
var (
disambLine = regexp.MustCompile(`^(\s*)disambiguate`)
definedRegs = make(map[string]string)
illegalCharacter = map[string]string{
"#": "Sharp",
"+": "Plus",
"-": "Dash",
}
)
type disambiguator struct {
Extension string `json:"extension,omitempty"`
Languages []*languageHeuristics `json:"languages,omitempty"`
}
func (d *disambiguator) setHeuristicsNames() {
for _, lang := range d.Languages {
for i, heuristic := range lang.Heuristics {
name := buildName(d.Extension, lang.Language, i)
heuristic.Name = name
}
}
}
func buildName(extension, language string, id int) string {
extension = strings.TrimPrefix(extension, `.`)
language = strings.Join(strings.Fields(language), ``)
name := strings.Join([]string{extension, language, "Matcher", strconv.Itoa(id)}, `_`)
for k, v := range illegalCharacter {
if strings.Contains(name, k) {
name = strings.Replace(name, k, v, -1)
}
}
return name
}
type languageHeuristics struct {
Language string `json:"language,omitempty"`
Heuristics []*heuristic `json:"heuristics,omitempty"`
LogicRelations []string `json:"logic_relations,omitempty"`
}
func (l *languageHeuristics) clone() (*languageHeuristics, error) {
language := l.Language
logicRels := make([]string, len(l.LogicRelations))
if copy(logicRels, l.LogicRelations) != len(l.LogicRelations) {
return nil, fmt.Errorf("error copying logic relations")
}
heuristics := make([]*heuristic, 0, len(l.Heuristics))
for _, h := range l.Heuristics {
heuristic := *h
heuristics = append(heuristics, &heuristic)
}
clone := &languageHeuristics{
Language: language,
Heuristics: heuristics,
LogicRelations: logicRels,
}
return clone, nil
}
type heuristic struct {
Name string `json:"name,omitempty"`
Regexp string `json:"regexp,omitempty"`
}
// A disambiguate block looks like:
// disambiguate ".mod", ".extension" do |data|
// if data.include?('<!ENTITY ') && data.include?('patata')
// Language["XML"]
// elsif /^\s*MODULE [\w\.]+;/i.match(data) || /^\s*END [\w\.]+;/i.match(data) || data.empty?
// Language["Modula-2"]
// elsif (/^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data))
// Language["Scala"]
// elsif (data.include?("gap> "))
// Language["GAP"]
// else
// [Language["Linux Kernel Module"], Language["AMPL"]]
// end
// end
func getDisambiguators(heuristics []byte) ([]*disambiguator, error) {
seenExtensions := map[string]bool{}
buf := bufio.NewScanner(bytes.NewReader(heuristics))
disambiguators := make([]*disambiguator, 0, 50)
for buf.Scan() {
line := buf.Text()
if disambLine.MatchString(line) {
d, err := parseDisambiguators(line, buf, seenExtensions)
if err != nil {
return nil, err
}
disambiguators = append(disambiguators, d...)
}
lookForRegexpVariables(line)
}
if err := buf.Err(); err != nil {
return nil, err
}
return disambiguators, nil
}
func lookForRegexpVariables(line string) {
if strings.Contains(line, "ObjectiveCRegex = ") {
line = strings.TrimSpace(line)
reg := strings.TrimPrefix(line, "ObjectiveCRegex = ")
definedRegs["ObjectiveCRegex"] = reg
}
if strings.Contains(line, "fortran_rx = ") {
line = strings.TrimSpace(line)
reg := strings.TrimPrefix(line, "fortran_rx = ")
definedRegs["fortran_rx"] = reg
}
}
func parseDisambiguators(line string, buf *bufio.Scanner, seenExtensions map[string]bool) ([]*disambiguator, error) {
disambList := make([]*disambiguator, 0, 2)
splitted := strings.Fields(line)
for _, v := range splitted {
if strings.HasPrefix(v, `"`) {
extension := strings.Trim(v, `",`)
if _, ok := seenExtensions[extension]; !ok {
d := &disambiguator{Extension: extension}
disambList = append(disambList, d)
seenExtensions[extension] = true
}
}
}
langsHeuristics, err := getLanguagesHeuristics(buf)
if err != nil {
return nil, err
}
for i, disamb := range disambList {
lh := langsHeuristics
if i != 0 {
lh = cloneLanguagesHeuristics(langsHeuristics)
}
disamb.Languages = lh
disamb.setHeuristicsNames()
}
return disambList, nil
}
func cloneLanguagesHeuristics(list []*languageHeuristics) []*languageHeuristics {
cloneList := make([]*languageHeuristics, 0, len(list))
for _, langHeu := range list {
clone, _ := langHeu.clone()
cloneList = append(cloneList, clone)
}
return cloneList
}
func getLanguagesHeuristics(buf *bufio.Scanner) ([]*languageHeuristics, error) {
langsList := make([][]string, 0, 2)
heuristicsList := make([][]*heuristic, 0, 1)
logicRelsList := make([][]string, 0, 1)
lastWasMatch := false
for buf.Scan() {
line := buf.Text()
if strings.TrimSpace(line) == "end" {
break
}
if hasRegExp(line) {
line := cleanRegExpLine(line)
logicRels := getLogicRelations(line)
heuristics := getHeuristics(line)
if lastWasMatch {
i := len(heuristicsList) - 1
heuristicsList[i] = append(heuristicsList[i], heuristics...)
i = len(logicRelsList) - 1
logicRelsList[i] = append(logicRelsList[i], logicRels...)
} else {
heuristicsList = append(heuristicsList, heuristics)
logicRelsList = append(logicRelsList, logicRels)
}
lastWasMatch = true
}
if strings.Contains(line, "Language") {
langs := getLanguages(line)
langsList = append(langsList, langs)
lastWasMatch = false
}
}
if err := buf.Err(); err != nil {
return nil, err
}
langsHeuristics := buildLanguagesHeuristics(langsList, heuristicsList, logicRelsList)
return langsHeuristics, nil
}
func hasRegExp(line string) bool {
return strings.Contains(line, ".match") || strings.Contains(line, ".include?") || strings.Contains(line, ".empty?")
}
func cleanRegExpLine(line string) string {
if strings.Contains(line, "if ") {
line = line[strings.Index(line, `if `)+3:]
}
line = strings.TrimSpace(line)
line = strings.TrimPrefix(line, `(`)
if strings.Contains(line, "))") {
line = strings.TrimSuffix(line, `)`)
}
return line
}
func getLogicRelations(line string) []string {
rels := make([]string, 0)
splitted := strings.Split(line, "||")
for i, v := range splitted {
if strings.Contains(v, "&&") {
rels = append(rels, "&&")
}
if i < len(splitted)-1 {
rels = append(rels, "||")
}
}
if len(rels) == 0 {
rels = nil
}
return rels
}
func getHeuristics(line string) []*heuristic {
splitted := splitByLogicOps(line)
heuristics := make([]*heuristic, 0, len(splitted))
for _, v := range splitted {
v = strings.TrimSpace(v)
var reg string
if strings.Contains(v, ".match") {
reg = v[:strings.Index(v, ".match")]
reg = replaceRegexpVariables(reg)
}
if strings.Contains(v, ".include?") {
reg = includeToRegExp(v)
}
if strings.Contains(v, ".empty?") {
reg = emptyFile
}
if reg != "" {
reg = convertToValidRegexp(reg)
heuristics = append(heuristics, &heuristic{Regexp: reg})
}
}
return heuristics
}
func splitByLogicOps(line string) []string {
splitted := make([]string, 0, 1)
splitOr := strings.Split(line, "||")
for _, v := range splitOr {
splitAnd := strings.Split(v, "&&")
splitted = append(splitted, splitAnd...)
}
return splitted
}
func replaceRegexpVariables(reg string) string {
repl := reg
if v, ok := definedRegs[reg]; ok {
repl = v
}
return repl
}
func convertToValidRegexp(reg string) string {
// example: `/^(\s*)(<Project|<Import|<Property|<?xml|xmlns)/i``
// Ruby modifier "m" matches multiple lines, recognizing newlines as normal characters, Go use flag "s" for that.
const (
caseSensitive = "i"
matchEOL = "s"
rubyCaseSensitive = "i"
rubyMultiLine = "m"
)
if reg == emptyFile {
return reg
}
reg = strings.TrimPrefix(reg, `/`)
flags := "(?m"
lastSlash := strings.LastIndex(reg, `/`)
if lastSlash == -1 {
return flags + ")" + reg
}
specialChars := reg[lastSlash:]
reg = reg[:lastSlash]
if lastSlash == len(reg)-1 {
return flags + ")" + reg
}
if strings.Contains(specialChars, rubyCaseSensitive) {
flags = flags + caseSensitive
}
if strings.Contains(specialChars, rubyMultiLine) {
flags = flags + matchEOL
}
return flags + ")" + reg
}
func includeToRegExp(include string) string {
content := include[strings.Index(include, `(`)+1 : strings.Index(include, `)`)]
content = strings.Trim(content, `"'`)
return regexp.QuoteMeta(content)
}
func getLanguages(line string) []string {
languages := make([]string, 0)
splitted := strings.Split(line, `,`)
for _, lang := range splitted {
lang = trimLanguage(lang)
languages = append(languages, lang)
}
return languages
}
func trimLanguage(enclosedLang string) string {
lang := strings.TrimSpace(enclosedLang)
lang = lang[strings.Index(lang, `"`)+1:]
lang = lang[:strings.Index(lang, `"`)]
return lang
}
func buildLanguagesHeuristics(langsList [][]string, heuristicsList [][]*heuristic, logicRelsList [][]string) []*languageHeuristics {
langsHeuristics := make([]*languageHeuristics, 0, len(langsList))
for i, langSlice := range langsList {
var heuristics []*heuristic
if i < len(heuristicsList) {
heuristics = heuristicsList[i]
}
var rels []string
if i < len(logicRelsList) {
rels = logicRelsList[i]
}
for _, lang := range langSlice {
lh := &languageHeuristics{
Language: lang,
Heuristics: heuristics,
LogicRelations: rels,
}
langsHeuristics = append(langsHeuristics, lh)
}
}
return langsHeuristics
}
func executeContentTemplate(out io.Writer, disambiguators []*disambiguator, tmplPath, tmplName, commit string) error {
fmap := template.FuncMap{
"getCommit": func() string { return commit },
"getAllHeuristics": getAllHeuristics,
"returnStringSlice": func(slice []string) string {
if len(slice) == 0 {
return "nil"
}
return `[]string{` + strings.Join(slice, `, `) + `}`
},
"returnLanguages": returnLanguages,
"avoidLanguage": avoidLanguage,
}
t := template.Must(template.New(tmplName).Funcs(fmap).ParseFiles(tmplPath))
if err := t.Execute(out, disambiguators); err != nil {
return err
}
return nil
}
func getAllHeuristics(disambiguators []*disambiguator) []*heuristic {
heuristics := make([]*heuristic, 0)
for _, disamb := range disambiguators {
for _, lang := range disamb.Languages {
if !avoidLanguage(lang) {
heuristics = append(heuristics, lang.Heuristics...)
}
}
}
return heuristics
}
func avoidLanguage(lang *languageHeuristics) bool {
// necessary to avoid corner cases
for _, heuristic := range lang.Heuristics {
if containsInvalidRegexp(heuristic.Regexp) {
return true
}
}
return false
}
func containsInvalidRegexp(reg string) bool {
return strings.Contains(reg, `(?<`) || strings.Contains(reg, `\1`)
}
func returnLanguages(langsHeuristics []*languageHeuristics) []string {
langs := make([]string, 0)
for _, langHeu := range langsHeuristics {
if len(langHeu.Heuristics) == 0 {
langs = append(langs, `"`+langHeu.Language+`"`)
}
}
return langs
}