tartrazine/internal/code-generator/generator/frequencies.go

203 lines
5.3 KiB
Go
Raw Normal View History

2017-05-25 10:33:26 +00:00
package generator
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"log"
"math"
"os"
2017-05-25 10:33:26 +00:00
"path/filepath"
"sort"
"strconv"
"strings"
2017-05-25 10:33:26 +00:00
"text/template"
2020-03-19 16:31:29 +00:00
"github.com/go-enry/go-enry/v2/internal/tokenizer"
2017-05-25 10:33:26 +00:00
)
type samplesFrequencies struct {
LanguageTotal int `json:"language_total,omitempty"`
Languages map[string]int `json:"languages,omitempty"`
TokensTotal int `json:"tokens_total,omitempty"`
Tokens map[string]map[string]int `json:"tokens,omitempty"`
LanguageTokens map[string]int `json:"language_tokens,omitempty"`
}
// Frequencies reads directories in samplesDir, retrieves information about frequencies of languages and tokens, and write
// the file outPath using tmplName as a template. It complies with type File signature.
2017-06-13 11:56:07 +00:00
func Frequencies(fileToParse, samplesDir, outPath, tmplPath, tmplName, commit string) error {
2017-05-25 10:33:26 +00:00
freqs, err := getFrequencies(samplesDir)
if err != nil {
return err
}
if _, ok := os.LookupEnv("ENRY_DEBUG"); ok {
log.Printf("Total samples: %d\n", freqs.LanguageTotal)
log.Printf("Total tokens: %d\n", freqs.TokensTotal)
keys := make([]string, 0, len(freqs.Languages))
for k := range freqs.Languages {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
fmt.Printf(" %s: %d\n", k, freqs.Languages[k])
}
}
2017-05-25 10:33:26 +00:00
buf := &bytes.Buffer{}
2017-06-13 11:56:07 +00:00
if err := executeFrequenciesTemplate(buf, freqs, tmplPath, tmplName, commit); err != nil {
2017-05-25 10:33:26 +00:00
return err
}
return formatedWrite(outPath, buf.Bytes())
2017-05-25 10:33:26 +00:00
}
func getFrequencies(samplesDir string) (*samplesFrequencies, error) {
langDirs, err := ioutil.ReadDir(samplesDir)
2017-05-25 10:33:26 +00:00
if err != nil {
return nil, err
}
var languageTotal int
var languages = make(map[string]int)
var tokensTotal int
var tokens = make(map[string]map[string]int)
var languageTokens = make(map[string]int)
for _, langDir := range langDirs {
if !langDir.IsDir() {
2017-05-25 10:33:26 +00:00
continue
}
lang := langDir.Name()
samples, err := readSamples(filepath.Join(samplesDir, lang))
2017-05-25 10:33:26 +00:00
if err != nil {
log.Println(err)
}
if len(samples) == 0 {
continue
}
samplesTokens, err := getTokens(samples)
if err != nil {
log.Println(err)
continue
}
languageTotal += len(samples)
languages[lang] = len(samples)
tokensTotal += len(samplesTokens)
languageTokens[lang] = len(samplesTokens)
tokens[lang] = make(map[string]int)
for _, token := range samplesTokens {
tokens[lang][token]++
}
}
return &samplesFrequencies{
TokensTotal: tokensTotal,
LanguageTotal: languageTotal,
Tokens: tokens,
LanguageTokens: languageTokens,
Languages: languages,
}, nil
}
// readSamples collects ./samples/ filenames from the Linguist codebase, skiping symlinks.
func readSamples(samplesLangDir string) ([]string, error) {
const specialSubDir = "filenames"
var samples []string
2017-05-25 10:33:26 +00:00
err := filepath.Walk(samplesLangDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
fmt.Printf("failure accessing a path %q: %v\n", path, err)
return err
}
if info.IsDir() {
switch info.Name() {
case filepath.Base(samplesLangDir):
return nil
case specialSubDir:
return nil
default:
return filepath.SkipDir
2017-05-25 10:33:26 +00:00
}
}
// skip git file symlinks on win and *nix
if isKnownSymlinkInLinguist(path) || !info.Mode().IsRegular() {
return nil
}
samples = append(samples, path)
return nil
})
2017-05-25 10:33:26 +00:00
return samples, err
2017-05-25 10:33:26 +00:00
}
// isKnownSymlinkInLinguist checks if the file name is on the list of known symlinks.
// On Windows, there is no symlink support in Git [1] and those become regular text files,
// so we have to skip these files manually, maintaing a list here :/
// 1. https://github.com/git-for-windows/git/wiki/Symbolic-Links
//
// $ find -L .linguist/samples -xtype l
func isKnownSymlinkInLinguist(path string) bool {
return strings.HasSuffix(path, filepath.Join("Ant Build System", "filenames", "build.xml")) ||
strings.HasSuffix(path, filepath.Join("Markdown", "symlink.md"))
2017-05-25 10:33:26 +00:00
}
func getTokens(samples []string) ([]string, error) {
tokens := make([]string, 0, 20)
var anyError error
for _, sample := range samples {
content, err := ioutil.ReadFile(sample)
if err != nil {
anyError = err
continue
}
t := tokenizer.Tokenize(content)
tokens = append(tokens, t...)
}
return tokens, anyError
}
2017-06-13 11:56:07 +00:00
func executeFrequenciesTemplate(out io.Writer, freqs *samplesFrequencies, tmplPath, tmplName, commit string) error {
2017-05-25 10:33:26 +00:00
fmap := template.FuncMap{
"toFloat64": func(num int) string { return fmt.Sprintf("%f", float64(num)) },
"orderKeys": func(m map[string]int) []string {
keys := make([]string, 0, len(m))
for key := range m {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
},
"languageLogProbability": func(language string) string {
num := math.Log(float64(freqs.Languages[language]) / float64(freqs.LanguageTotal))
return fmt.Sprintf("%f", num)
},
"orderMapMapKeys": func(mm map[string]map[string]int) []string {
keys := make([]string, 0, len(mm))
for key := range mm {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
},
"tokenLogProbability": func(language, token string) string {
num := math.Log(float64(freqs.Tokens[language][token]) / float64(freqs.LanguageTokens[language]))
return fmt.Sprintf("%f", num)
},
"quote": strconv.Quote,
}
return executeTemplate(out, tmplName, tmplPath, commit, fmap, freqs)
2017-05-25 10:33:26 +00:00
}