diff --git a/classifier.go b/classifier.go index 39b6ea7..ab50ee8 100644 --- a/classifier.go +++ b/classifier.go @@ -2,15 +2,15 @@ package enry import ( "math" + "sort" "gopkg.in/src-d/enry.v1/internal/tokenizer" ) -// Classifier is the interface that contains the method Classify which is in charge to assign scores to the possibles candidates. -// The scores must order the candidates so as the highest score be the most probably language of the content. The candidates is -// a map which can be used to assign weights to languages dynamically. +// Classifier is the interface in charge to detect the possible languages of the given content based on a set of +// candidates. Candidates is a map which can be used to assign weights to languages dynamically. type Classifier interface { - Classify(content []byte, candidates map[string]float64) map[string]float64 + Classify(content []byte, candidates map[string]float64) (languages []string) } type classifier struct { @@ -19,7 +19,13 @@ type classifier struct { tokensTotal float64 } -func (c *classifier) Classify(content []byte, candidates map[string]float64) map[string]float64 { +type scoredLanguage struct { + language string + score float64 +} + +// Classify returns a sorted slice of possible languages sorted by decreasing language's probability +func (c *classifier) Classify(content []byte, candidates map[string]float64) []string { if len(content) == 0 { return nil } @@ -31,18 +37,35 @@ func (c *classifier) Classify(content []byte, candidates map[string]float64) map languages = make(map[string]float64, len(candidates)) for candidate, weight := range candidates { if lang, ok := GetLanguageByAlias(candidate); ok { - languages[lang] = weight + candidate = lang } + + languages[candidate] = weight } } tokens := tokenizer.Tokenize(content) - scores := make(map[string]float64, len(languages)) + scoredLangs := make([]*scoredLanguage, 0, len(languages)) for language := range languages { - scores[language] = c.tokensLogProbability(tokens, language) + c.languagesLogProbabilities[language] + scoredLang := &scoredLanguage{ + language: language, + score: c.tokensLogProbability(tokens, language) + c.languagesLogProbabilities[language], + } + + scoredLangs = append(scoredLangs, scoredLang) } - return scores + return sortLanguagesByScore(scoredLangs) +} + +func sortLanguagesByScore(scoredLangs []*scoredLanguage) []string { + sort.SliceStable(scoredLangs, func(i, j int) bool { return scoredLangs[j].score < scoredLangs[i].score }) + sortedLanguages := make([]string, 0, len(scoredLangs)) + for _, scoredLang := range scoredLangs { + sortedLanguages = append(sortedLanguages, scoredLang.language) + } + + return sortedLanguages } func (c *classifier) knownLangs() map[string]float64 { diff --git a/common.go b/common.go index a4615e4..dd6f4b5 100644 --- a/common.go +++ b/common.go @@ -1,96 +1,371 @@ package enry import ( - "math" + "bufio" + "bytes" "path/filepath" + "regexp" "strings" ) // OtherLanguage is used as a zero value when a function can not return a specific language. -const OtherLanguage = "Other" +const OtherLanguage = "" // Strategy type fix the signature for the functions that can be used as a strategy. -type Strategy func(filename string, content []byte) (languages []string) +type Strategy func(filename string, content []byte, candidates []string) (languages []string) -var strategies = []Strategy{ +// DefaultStrategies is the strategies' sequence GetLanguage uses to detect languages. +var DefaultStrategies = []Strategy{ GetLanguagesByModeline, GetLanguagesByFilename, GetLanguagesByShebang, GetLanguagesByExtension, GetLanguagesByContent, + GetLanguagesByClassifier, } // GetLanguage applies a sequence of strategies based on the given filename and content // to find out the most probably language to return. -func GetLanguage(filename string, content []byte) string { - candidates := map[string]float64{} - for _, strategy := range strategies { - languages := strategy(filename, content) +func GetLanguage(filename string, content []byte) (language string) { + languages := GetLanguages(filename, content) + return firstLanguage(languages) +} + +func firstLanguage(languages []string) string { + if len(languages) == 0 { + return OtherLanguage + } + + return languages[0] +} + +// GetLanguageByModeline returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByModeline(content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByModeline, "", content, nil) +} + +// GetLanguageByEmacsModeline returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByEmacsModeline(content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByEmacsModeline, "", content, nil) +} + +// GetLanguageByVimModeline returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByVimModeline(content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByVimModeline, "", content, nil) +} + +// GetLanguageByFilename returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByFilename(filename string) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByFilename, filename, nil, nil) +} + +// GetLanguageByShebang returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByShebang(content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByShebang, "", content, nil) +} + +// GetLanguageByExtension returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByExtension(filename string) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByExtension, filename, nil, nil) +} + +// GetLanguageByContent returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByContent(content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByContent, "", content, nil) +} + +// GetLanguageByClassifier returns the most probably language detected for the given content. It uses +// DefaultClassifier, if no candidates are provided it returns OtherLanguage. +func GetLanguageByClassifier(content []byte, candidates []string) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByClassifier, "", content, candidates) +} + +func getLanguageByStrategy(strategy Strategy, filename string, content []byte, candidates []string) (string, bool) { + languages := strategy(filename, content, candidates) + return getFirstLanguageAndSafe(languages) +} + +func getFirstLanguageAndSafe(languages []string) (language string, safe bool) { + language = firstLanguage(languages) + safe = len(languages) == 1 + return +} + +// GetLanguageBySpecificClassifier returns the most probably language for the given content using +// classifier to detect language. +func GetLanguageBySpecificClassifier(content []byte, candidates []string, classifier Classifier) (language string, safe bool) { + languages := GetLanguagesBySpecificClassifier(content, candidates, classifier) + return getFirstLanguageAndSafe(languages) +} + +// GetLanguages applies a sequence of strategies based on the given filename and content +// to find out the most probably languages to return. +func GetLanguages(filename string, content []byte) []string { + var languages []string + candidates := []string{} + for _, strategy := range DefaultStrategies { + languages = strategy(filename, content, candidates) if len(languages) == 1 { - return languages[0] + return languages } if len(languages) > 0 { - for _, language := range languages { - candidates[language]++ + candidates = append(candidates, languages...) + } + } + + return languages +} + +// GetLanguagesByModeline returns a slice of possible languages for the given content, filename will be ignored. +// It is comply with the signature to be a Strategy type. +func GetLanguagesByModeline(filename string, content []byte, candidates []string) []string { + headFoot := getHeaderAndFooter(content) + var languages []string + for _, getLang := range modelinesFunc { + languages = getLang("", headFoot, candidates) + if len(languages) > 0 { + break + } + } + + return languages +} + +var modelinesFunc = []Strategy{ + GetLanguagesByEmacsModeline, + GetLanguagesByVimModeline, +} + +func getHeaderAndFooter(content []byte) []byte { + const searchScope = 5 + if bytes.Count(content, []byte("\n")) < 2*searchScope { + return content + } + + header := headScope(content, searchScope) + footer := footScope(content, searchScope) + headerAndFooter := make([]byte, 0, len(content[:header])+len(content[footer:])) + headerAndFooter = append(headerAndFooter, content[:header]...) + headerAndFooter = append(headerAndFooter, content[footer:]...) + return headerAndFooter +} + +func headScope(content []byte, scope int) (index int) { + for i := 0; i < scope; i++ { + eol := bytes.IndexAny(content, "\n") + content = content[eol+1:] + index += eol + } + + return index + scope - 1 +} + +func footScope(content []byte, scope int) (index int) { + for i := 0; i < scope; i++ { + index = bytes.LastIndexAny(content, "\n") + content = content[:index] + } + + return index + 1 +} + +var ( + reEmacsModeline = regexp.MustCompile(`.*-\*-\s*(.+?)\s*-\*-.*(?m:$)`) + reEmacsLang = regexp.MustCompile(`.*(?i:mode)\s*:\s*([^\s;]+)\s*;*.*`) + reVimModeline = regexp.MustCompile(`(?:(?m:\s|^)vi(?:m[<=>]?\d+|m)?|[\t\x20]*ex)\s*[:]\s*(.*)(?m:$)`) + reVimLang = regexp.MustCompile(`(?i:filetype|ft|syntax)\s*=(\w+)(?:\s|:|$)`) +) + +// GetLanguagesByEmacsModeline returns a slice of possible languages for the given content, filename and candidates +// will be ignored. It is comply with the signature to be a Strategy type. +func GetLanguagesByEmacsModeline(filename string, content []byte, candidates []string) []string { + matched := reEmacsModeline.FindAllSubmatch(content, -1) + if matched == nil { + return nil + } + + // only take the last matched line, discard previous lines + lastLineMatched := matched[len(matched)-1][1] + matchedAlias := reEmacsLang.FindSubmatch(lastLineMatched) + var alias string + if matchedAlias != nil { + alias = string(matchedAlias[1]) + } else { + alias = string(lastLineMatched) + } + + language, ok := GetLanguageByAlias(alias) + if !ok { + return nil + } + + return []string{language} +} + +// GetLanguagesByVimModeline returns a slice of possible languages for the given content, filename and candidates +// will be ignored. It is comply with the signature to be a Strategy type. +func GetLanguagesByVimModeline(filename string, content []byte, candidates []string) []string { + matched := reVimModeline.FindAllSubmatch(content, -1) + if matched == nil { + return nil + } + + // only take the last matched line, discard previous lines + lastLineMatched := matched[len(matched)-1][1] + matchedAlias := reVimLang.FindAllSubmatch(lastLineMatched, -1) + if matchedAlias == nil { + return nil + } + + alias := string(matchedAlias[0][1]) + if len(matchedAlias) > 1 { + // cases: + // matchedAlias = [["syntax=ruby " "ruby"] ["ft=python " "python"] ["filetype=perl " "perl"]] returns OtherLanguage; + // matchedAlias = [["syntax=python " "python"] ["ft=python " "python"] ["filetype=python " "python"]] returns "Python"; + for _, match := range matchedAlias { + otherAlias := string(match[1]) + if otherAlias != alias { + return nil } } } - if len(candidates) == 0 { - return OtherLanguage + language, ok := GetLanguageByAlias(alias) + if !ok { + return nil } - lang := GetLanguageByClassifier(content, candidates, nil) - return lang + return []string{language} } -// GetLanguageByModeline returns the language of the given content looking for the modeline, -// and safe to indicate the sureness of returned language. -func GetLanguageByModeline(content []byte) (lang string, safe bool) { - return getLangAndSafe("", content, GetLanguagesByModeline) +// GetLanguagesByFilename returns a slice of possible languages for the given filename, content and candidates +// will be ignored. It is comply with the signature to be a Strategy type. +func GetLanguagesByFilename(filename string, content []byte, candidates []string) []string { + return languagesByFilename[filepath.Base(filename)] } -// GetLanguageByFilename returns a language based on the given filename, and safe to indicate -// the sureness of returned language. -func GetLanguageByFilename(filename string) (lang string, safe bool) { - return getLangAndSafe(filename, nil, GetLanguagesByFilename) +// GetLanguagesByShebang returns a slice of possible languages for the given content, filename and candidates +// will be ignored. It is comply with the signature to be a Strategy type. +func GetLanguagesByShebang(filename string, content []byte, candidates []string) (languages []string) { + interpreter := getInterpreter(content) + return languagesByInterpreter[interpreter] } -// GetLanguagesByFilename returns a slice of possible languages for the given filename, content will be ignored. -// It accomplish the signature to be a Strategy type. -func GetLanguagesByFilename(filename string, content []byte) []string { - return languagesByFilename[filename] +var ( + shebangExecHack = regexp.MustCompile(`exec (\w+).+\$0.+\$@`) + pythonVersion = regexp.MustCompile(`python\d\.\d+`) +) + +func getInterpreter(data []byte) (interpreter string) { + line := getFirstLine(data) + if !hasShebang(line) { + return "" + } + + // skip shebang + line = bytes.TrimSpace(line[2:]) + + splitted := bytes.Fields(line) + if bytes.Contains(splitted[0], []byte("env")) { + if len(splitted) > 1 { + interpreter = string(splitted[1]) + } + } else { + + splittedPath := bytes.Split(splitted[0], []byte{'/'}) + interpreter = string(splittedPath[len(splittedPath)-1]) + } + + if interpreter == "sh" { + interpreter = lookForMultilineExec(data) + } + + if pythonVersion.MatchString(interpreter) { + interpreter = interpreter[:strings.Index(interpreter, `.`)] + } + + return } -// GetLanguageByShebang returns the language of the given content looking for the shebang line, -// and safe to indicate the sureness of returned language. -func GetLanguageByShebang(content []byte) (lang string, safe bool) { - return getLangAndSafe("", content, GetLanguagesByShebang) +func getFirstLine(data []byte) []byte { + buf := bufio.NewScanner(bytes.NewReader(data)) + buf.Scan() + line := buf.Bytes() + if err := buf.Err(); err != nil { + return nil + } + + return line } -// GetLanguageByExtension returns a language based on the given filename, and safe to indicate -// the sureness of returned language. -func GetLanguageByExtension(filename string) (lang string, safe bool) { - return getLangAndSafe(filename, nil, GetLanguagesByExtension) +func hasShebang(line []byte) bool { + const shebang = `#!` + prefix := []byte(shebang) + return bytes.HasPrefix(line, prefix) } -// GetLanguagesByExtension returns a slice of possible languages for the given filename, content will be ignored. -// It accomplish the signature to be a Strategy type. -func GetLanguagesByExtension(filename string, content []byte) []string { - ext := strings.ToLower(filepath.Ext(filename)) - return languagesByExtension[ext] +func lookForMultilineExec(data []byte) string { + const magicNumOfLines = 5 + interpreter := "sh" + + buf := bufio.NewScanner(bytes.NewReader(data)) + for i := 0; i < magicNumOfLines && buf.Scan(); i++ { + line := buf.Bytes() + if shebangExecHack.Match(line) { + interpreter = shebangExecHack.FindStringSubmatch(string(line))[1] + break + } + } + + if err := buf.Err(); err != nil { + return interpreter + } + + return interpreter } -// GetLanguageByContent returns a language based on the filename and heuristics applies to the content, -// and safe to indicate the sureness of returned language. -func GetLanguageByContent(filename string, content []byte) (lang string, safe bool) { - return getLangAndSafe(filename, content, GetLanguagesByContent) +// GetLanguagesByExtension returns a slice of possible languages for the given filename, content and candidates +// will be ignored. It is comply with the signature to be a Strategy type. +func GetLanguagesByExtension(filename string, content []byte, candidates []string) []string { + if !strings.Contains(filename, ".") { + return nil + } + + filename = strings.ToLower(filename) + dots := getDotIndexes(filename) + for _, dot := range dots { + ext := filename[dot:] + languages, ok := languagesByExtension[ext] + if ok { + return languages + } + } + + return nil } -// GetLanguagesByContent returns a slice of possible languages for the given content, filename will be ignored. -// It accomplish the signature to be a Strategy type. -func GetLanguagesByContent(filename string, content []byte) []string { +func getDotIndexes(filename string) []int { + dots := make([]int, 0, 2) + for i, letter := range filename { + if letter == rune('.') { + dots = append(dots, i) + } + } + + return dots +} + +// GetLanguagesByContent returns a slice of possible languages for the given content, filename and candidates +// will be ignored. It is comply with the signature to be a Strategy type. +func GetLanguagesByContent(filename string, content []byte, candidates []string) []string { ext := strings.ToLower(filepath.Ext(filename)) fnMatcher, ok := contentMatchers[ext] if !ok { @@ -100,51 +375,24 @@ func GetLanguagesByContent(filename string, content []byte) []string { return fnMatcher(content) } -func getLangAndSafe(filename string, content []byte, getLanguageByStrategy Strategy) (lang string, safe bool) { - languages := getLanguageByStrategy(filename, content) - if len(languages) == 0 { - lang = OtherLanguage - return +// GetLanguagesByClassifier uses DefaultClassifier as a Classifier and returns a sorted slice of possible languages ordered by +// decreasing language's probability. If there are not candidates it returns nil. It is comply with the signature to be a Strategy type. +func GetLanguagesByClassifier(filename string, content []byte, candidates []string) (languages []string) { + if len(candidates) == 0 { + return nil } - lang = languages[0] - safe = len(languages) == 1 - return + return GetLanguagesBySpecificClassifier(content, candidates, DefaultClassifier) } -// GetLanguageByClassifier takes in a content and a list of candidates, and apply the classifier's Classify method to -// get the most probably language. If classifier is null then DefaultClassfier will be used. If there aren't candidates -// OtherLanguage is returned. -func GetLanguageByClassifier(content []byte, candidates map[string]float64, classifier Classifier) string { - scores := GetLanguagesByClassifier(content, candidates, classifier) - if len(scores) == 0 { - return OtherLanguage +// GetLanguagesBySpecificClassifier returns a slice of possible languages. It takes in a Classifier to be used. +func GetLanguagesBySpecificClassifier(content []byte, candidates []string, classifier Classifier) (languages []string) { + mapCandidates := make(map[string]float64) + for _, candidate := range candidates { + mapCandidates[candidate]++ } - return getLangugeHigherScore(scores) -} - -func getLangugeHigherScore(scores map[string]float64) string { - var language string - higher := -math.MaxFloat64 - for lang, score := range scores { - if higher < score { - language = lang - higher = score - } - } - - return language -} - -// GetLanguagesByClassifier returns a map of possible languages as keys and a score as value based on content and candidates. The values can be ordered -// with the highest value as the most probably language. If classifier is null then DefaultClassfier will be used. -func GetLanguagesByClassifier(content []byte, candidates map[string]float64, classifier Classifier) map[string]float64 { - if classifier == nil { - classifier = DefaultClassifier - } - - return classifier.Classify(content, candidates) + return classifier.Classify(content, mapCandidates) } // GetLanguageExtensions returns the different extensions being used by the language. @@ -164,7 +412,7 @@ const ( Prose ) -// GetLanguageType returns the given language's type. +// GetLanguageType returns the type of the given language. func GetLanguageType(language string) (langType Type) { langType, ok := languagesType[language] if !ok { diff --git a/common_test.go b/common_test.go index 3a0b779..8ae456c 100644 --- a/common_test.go +++ b/common_test.go @@ -6,7 +6,6 @@ import ( "os" "path/filepath" "testing" - "text/tabwriter" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" @@ -26,6 +25,7 @@ func (s *SimpleLinguistTestSuite) TestGetLanguage() { filename string content []byte expected string + safe bool }{ {name: "TestGetLanguage_1", filename: "foo.py", content: []byte{}, expected: "Python"}, {name: "TestGetLanguage_2", filename: "foo.m", content: []byte(":- module"), expected: "Mercury"}, @@ -38,65 +38,64 @@ func (s *SimpleLinguistTestSuite) TestGetLanguage() { } } -func (s *SimpleLinguistTestSuite) TestGetLanguageByModelineLinguist() { +func (s *SimpleLinguistTestSuite) TestGetLanguagesByModelineLinguist() { const ( modelinesDir = ".linguist/test/fixtures/Data/Modelines" samplesDir = ".linguist/samples" ) tests := []struct { - name string - filename string - expectedLang string - expectedSafe bool + name string + filename string + candidates []string + expected []string }{ // Emacs - {name: "TestGetLanguageByModelineLinguist_1", filename: filepath.Join(modelinesDir, "example_smalltalk.md"), expectedLang: "Smalltalk", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_2", filename: filepath.Join(modelinesDir, "fundamentalEmacs.c"), expectedLang: "Text", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_3", filename: filepath.Join(modelinesDir, "iamphp.inc"), expectedLang: "PHP", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_4", filename: filepath.Join(modelinesDir, "seeplusplusEmacs1"), expectedLang: "C++", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_5", filename: filepath.Join(modelinesDir, "seeplusplusEmacs2"), expectedLang: "C++", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_6", filename: filepath.Join(modelinesDir, "seeplusplusEmacs3"), expectedLang: "C++", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_7", filename: filepath.Join(modelinesDir, "seeplusplusEmacs4"), expectedLang: "C++", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_8", filename: filepath.Join(modelinesDir, "seeplusplusEmacs5"), expectedLang: "C++", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_9", filename: filepath.Join(modelinesDir, "seeplusplusEmacs6"), expectedLang: "C++", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_10", filename: filepath.Join(modelinesDir, "seeplusplusEmacs7"), expectedLang: "C++", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_11", filename: filepath.Join(modelinesDir, "seeplusplusEmacs9"), expectedLang: "C++", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_12", filename: filepath.Join(modelinesDir, "seeplusplusEmacs10"), expectedLang: "C++", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_13", filename: filepath.Join(modelinesDir, "seeplusplusEmacs11"), expectedLang: "C++", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_14", filename: filepath.Join(modelinesDir, "seeplusplusEmacs12"), expectedLang: "C++", expectedSafe: true}, + {name: "TestGetLanguagesByModelineLinguist_1", filename: filepath.Join(modelinesDir, "example_smalltalk.md"), expected: []string{"Smalltalk"}}, + {name: "TestGetLanguagesByModelineLinguist_2", filename: filepath.Join(modelinesDir, "fundamentalEmacs.c"), expected: []string{"Text"}}, + {name: "TestGetLanguagesByModelineLinguist_3", filename: filepath.Join(modelinesDir, "iamphp.inc"), expected: []string{"PHP"}}, + {name: "TestGetLanguagesByModelineLinguist_4", filename: filepath.Join(modelinesDir, "seeplusplusEmacs1"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_5", filename: filepath.Join(modelinesDir, "seeplusplusEmacs2"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_6", filename: filepath.Join(modelinesDir, "seeplusplusEmacs3"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_7", filename: filepath.Join(modelinesDir, "seeplusplusEmacs4"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_8", filename: filepath.Join(modelinesDir, "seeplusplusEmacs5"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_9", filename: filepath.Join(modelinesDir, "seeplusplusEmacs6"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_10", filename: filepath.Join(modelinesDir, "seeplusplusEmacs7"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_11", filename: filepath.Join(modelinesDir, "seeplusplusEmacs9"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_12", filename: filepath.Join(modelinesDir, "seeplusplusEmacs10"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_13", filename: filepath.Join(modelinesDir, "seeplusplusEmacs11"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_14", filename: filepath.Join(modelinesDir, "seeplusplusEmacs12"), expected: []string{"C++"}}, // Vim - {name: "TestGetLanguageByModelineLinguist_15", filename: filepath.Join(modelinesDir, "seeplusplus"), expectedLang: "C++", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_16", filename: filepath.Join(modelinesDir, "iamjs.pl"), expectedLang: "JavaScript", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_17", filename: filepath.Join(modelinesDir, "iamjs2.pl"), expectedLang: "JavaScript", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_18", filename: filepath.Join(modelinesDir, "not_perl.pl"), expectedLang: "Prolog", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_19", filename: filepath.Join(modelinesDir, "ruby"), expectedLang: "Ruby", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_20", filename: filepath.Join(modelinesDir, "ruby2"), expectedLang: "Ruby", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_21", filename: filepath.Join(modelinesDir, "ruby3"), expectedLang: "Ruby", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_22", filename: filepath.Join(modelinesDir, "ruby4"), expectedLang: "Ruby", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_23", filename: filepath.Join(modelinesDir, "ruby5"), expectedLang: "Ruby", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_24", filename: filepath.Join(modelinesDir, "ruby6"), expectedLang: "Ruby", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_25", filename: filepath.Join(modelinesDir, "ruby7"), expectedLang: "Ruby", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_26", filename: filepath.Join(modelinesDir, "ruby8"), expectedLang: "Ruby", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_27", filename: filepath.Join(modelinesDir, "ruby9"), expectedLang: "Ruby", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_28", filename: filepath.Join(modelinesDir, "ruby10"), expectedLang: "Ruby", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_29", filename: filepath.Join(modelinesDir, "ruby11"), expectedLang: "Ruby", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_30", filename: filepath.Join(modelinesDir, "ruby12"), expectedLang: "Ruby", expectedSafe: true}, - {name: "TestGetLanguageByModelineLinguist_31", filename: filepath.Join(samplesDir, "C/main.c"), expectedLang: OtherLanguage, expectedSafe: false}, + {name: "TestGetLanguagesByModelineLinguist_15", filename: filepath.Join(modelinesDir, "seeplusplus"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_16", filename: filepath.Join(modelinesDir, "iamjs.pl"), expected: []string{"JavaScript"}}, + {name: "TestGetLanguagesByModelineLinguist_17", filename: filepath.Join(modelinesDir, "iamjs2.pl"), expected: []string{"JavaScript"}}, + {name: "TestGetLanguagesByModelineLinguist_18", filename: filepath.Join(modelinesDir, "not_perl.pl"), expected: []string{"Prolog"}}, + {name: "TestGetLanguagesByModelineLinguist_19", filename: filepath.Join(modelinesDir, "ruby"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_20", filename: filepath.Join(modelinesDir, "ruby2"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_21", filename: filepath.Join(modelinesDir, "ruby3"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_22", filename: filepath.Join(modelinesDir, "ruby4"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_23", filename: filepath.Join(modelinesDir, "ruby5"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_24", filename: filepath.Join(modelinesDir, "ruby6"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_25", filename: filepath.Join(modelinesDir, "ruby7"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_26", filename: filepath.Join(modelinesDir, "ruby8"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_27", filename: filepath.Join(modelinesDir, "ruby9"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_28", filename: filepath.Join(modelinesDir, "ruby10"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_29", filename: filepath.Join(modelinesDir, "ruby11"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_30", filename: filepath.Join(modelinesDir, "ruby12"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_31", filename: filepath.Join(samplesDir, "C/main.c"), expected: nil}, } for _, test := range tests { content, err := ioutil.ReadFile(test.filename) assert.NoError(s.T(), err) - lang, safe := GetLanguageByModeline(content) - assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang)) - assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe)) + languages := GetLanguagesByModeline(test.filename, content, test.candidates) + assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected)) } } -func (s *SimpleLinguistTestSuite) TestGetLanguageByModeline() { +func (s *SimpleLinguistTestSuite) TestGetLanguagesByModeline() { const ( wrongVim = `# vim: set syntax=ruby ft =python filetype=perl :` rightVim = `/* vim: set syntax=python ft =python filetype=python */` @@ -104,48 +103,48 @@ func (s *SimpleLinguistTestSuite) TestGetLanguageByModeline() { ) tests := []struct { - name string - content []byte - expectedLang string - expectedSafe bool + name string + filename string + content []byte + candidates []string + expected []string }{ - {name: "TestGetLanguageByModeline_1", content: []byte(wrongVim), expectedLang: OtherLanguage, expectedSafe: false}, - {name: "TestGetLanguageByModeline_2", content: []byte(rightVim), expectedLang: "Python", expectedSafe: true}, - {name: "TestGetLanguageByModeline_3", content: []byte(noLangVim), expectedLang: OtherLanguage, expectedSafe: false}, + {name: "TestGetLanguagesByModeline_1", content: []byte(wrongVim), expected: nil}, + {name: "TestGetLanguagesByModeline_2", content: []byte(rightVim), expected: []string{"Python"}}, + {name: "TestGetLanguagesByModeline_3", content: []byte(noLangVim), expected: nil}, } for _, test := range tests { - lang, safe := GetLanguageByModeline(test.content) - assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang)) - assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe)) + languages := GetLanguagesByModeline(test.filename, test.content, test.candidates) + assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected)) } } -func (s *SimpleLinguistTestSuite) TestGetLanguageByFilename() { +func (s *SimpleLinguistTestSuite) TestGetLanguagesByFilename() { tests := []struct { - name string - filename string - expectedLang string - expectedSafe bool + name string + filename string + content []byte + candidates []string + expected []string }{ - {name: "TestGetLanguageByFilename_1", filename: "unknown.interpreter", expectedLang: OtherLanguage, expectedSafe: false}, - {name: "TestGetLanguageByFilename_2", filename: ".bashrc", expectedLang: "Shell", expectedSafe: true}, - {name: "TestGetLanguageByFilename_3", filename: "Dockerfile", expectedLang: "Dockerfile", expectedSafe: true}, - {name: "TestGetLanguageByFilename_4", filename: "Makefile.frag", expectedLang: "Makefile", expectedSafe: true}, - {name: "TestGetLanguageByFilename_5", filename: "makefile", expectedLang: "Makefile", expectedSafe: true}, - {name: "TestGetLanguageByFilename_6", filename: "Vagrantfile", expectedLang: "Ruby", expectedSafe: true}, - {name: "TestGetLanguageByFilename_7", filename: "_vimrc", expectedLang: "Vim script", expectedSafe: true}, - {name: "TestGetLanguageByFilename_8", filename: "pom.xml", expectedLang: "Maven POM", expectedSafe: true}, + {name: "TestGetLanguagesByFilename_1", filename: "unknown.interpreter", expected: nil}, + {name: "TestGetLanguagesByFilename_2", filename: ".bashrc", expected: []string{"Shell"}}, + {name: "TestGetLanguagesByFilename_3", filename: "Dockerfile", expected: []string{"Dockerfile"}}, + {name: "TestGetLanguagesByFilename_4", filename: "Makefile.frag", expected: []string{"Makefile"}}, + {name: "TestGetLanguagesByFilename_5", filename: "makefile", expected: []string{"Makefile"}}, + {name: "TestGetLanguagesByFilename_6", filename: "Vagrantfile", expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByFilename_7", filename: "_vimrc", expected: []string{"Vim script"}}, + {name: "TestGetLanguagesByFilename_8", filename: "pom.xml", expected: []string{"Maven POM"}}, } for _, test := range tests { - lang, safe := GetLanguageByFilename(test.filename) - assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang)) - assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe)) + languages := GetLanguagesByFilename(test.filename, test.content, test.candidates) + assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected)) } } -func (s *SimpleLinguistTestSuite) TestGetLanguageByShebang() { +func (s *SimpleLinguistTestSuite) TestGetLanguagesByShebang() { const ( multilineExecHack = `#!/bin/sh # Next line is comment in Tcl, but not in sh... \ @@ -162,125 +161,112 @@ println("The shell script says ",vm.arglist.concat(" "));` ) tests := []struct { - name string - content []byte - expectedLang string - expectedSafe bool + name string + filename string + content []byte + candidates []string + expected []string }{ - {name: "TestGetLanguageByShebang_1", content: []byte(`#!/unknown/interpreter`), expectedLang: OtherLanguage, expectedSafe: false}, - {name: "TestGetLanguageByShebang_2", content: []byte(`no shebang`), expectedLang: OtherLanguage, expectedSafe: false}, - {name: "TestGetLanguageByShebang_3", content: []byte(`#!/usr/bin/env`), expectedLang: OtherLanguage, expectedSafe: false}, - {name: "TestGetLanguageByShebang_4", content: []byte(`#!/usr/bin/python -tt`), expectedLang: "Python", expectedSafe: true}, - {name: "TestGetLanguageByShebang_5", content: []byte(`#!/usr/bin/env python2.6`), expectedLang: "Python", expectedSafe: true}, - {name: "TestGetLanguageByShebang_6", content: []byte(`#!/usr/bin/env perl`), expectedLang: "Perl", expectedSafe: true}, - {name: "TestGetLanguageByShebang_7", content: []byte(`#! /bin/sh`), expectedLang: "Shell", expectedSafe: true}, - {name: "TestGetLanguageByShebang_8", content: []byte(`#!bash`), expectedLang: "Shell", expectedSafe: true}, - {name: "TestGetLanguageByShebang_9", content: []byte(multilineExecHack), expectedLang: "Tcl", expectedSafe: true}, - {name: "TestGetLanguageByShebang_10", content: []byte(multilineNoExecHack), expectedLang: "Shell", expectedSafe: true}, + {name: "TestGetLanguagesByShebang_1", content: []byte(`#!/unknown/interpreter`), expected: nil}, + {name: "TestGetLanguagesByShebang_2", content: []byte(`no shebang`), expected: nil}, + {name: "TestGetLanguagesByShebang_3", content: []byte(`#!/usr/bin/env`), expected: nil}, + {name: "TestGetLanguagesByShebang_4", content: []byte(`#!/usr/bin/python -tt`), expected: []string{"Python"}}, + {name: "TestGetLanguagesByShebang_5", content: []byte(`#!/usr/bin/env python2.6`), expected: []string{"Python"}}, + {name: "TestGetLanguagesByShebang_6", content: []byte(`#!/usr/bin/env perl`), expected: []string{"Perl"}}, + {name: "TestGetLanguagesByShebang_7", content: []byte(`#! /bin/sh`), expected: []string{"Shell"}}, + {name: "TestGetLanguagesByShebang_8", content: []byte(`#!bash`), expected: []string{"Shell"}}, + {name: "TestGetLanguagesByShebang_9", content: []byte(multilineExecHack), expected: []string{"Tcl"}}, + {name: "TestGetLanguagesByShebang_10", content: []byte(multilineNoExecHack), expected: []string{"Shell"}}, } for _, test := range tests { - lang, safe := GetLanguageByShebang(test.content) - assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang)) - assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe)) + languages := GetLanguagesByShebang(test.filename, test.content, test.candidates) + assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected)) } } -func (s *SimpleLinguistTestSuite) TestGetLanguageByExtension() { +func (s *SimpleLinguistTestSuite) TestGetLanguagesByExtension() { tests := []struct { - name string - filename string - expectedLang string - expectedSafe bool + name string + filename string + content []byte + candidates []string + expected []string }{ - {name: "TestGetLanguageByExtension_1", filename: "foo.foo", expectedLang: OtherLanguage, expectedSafe: false}, - {name: "TestGetLanguageByExtension_2", filename: "foo.go", expectedLang: "Go", expectedSafe: true}, - {name: "TestGetLanguageByExtension_3", filename: "foo.go.php", expectedLang: "Hack", expectedSafe: false}, + {name: "TestGetLanguagesByExtension_1", filename: "foo.foo", expected: nil}, + {name: "TestGetLanguagesByExtension_2", filename: "foo.go", expected: []string{"Go"}}, + {name: "TestGetLanguagesByExtension_3", filename: "foo.go.php", expected: []string{"Hack", "PHP"}}, } for _, test := range tests { - lang, safe := GetLanguageByExtension(test.filename) - assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang)) - assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe)) + languages := GetLanguagesByExtension(test.filename, test.content, test.candidates) + assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected)) } } -func (s *SimpleLinguistTestSuite) TestGetLanguageByContentLinguistCorpus() { - var total, failed, ok, other, unsafe int - - w := new(tabwriter.Writer) - w.Init(os.Stdout, 0, 8, 0, '\t', 0) - - filepath.Walk(".linguist/samples", func(path string, f os.FileInfo, err error) error { - if f.IsDir() { - if f.Name() == "filenames" { - return filepath.SkipDir - } - - return nil - } - - expected := filepath.Base(filepath.Dir(path)) - filename := filepath.Base(path) - extension := filepath.Ext(path) - content, _ := ioutil.ReadFile(path) - - if extension == "" { - return nil - } - - total++ - obtained, safe := GetLanguageByContent(filename, content) - if obtained == OtherLanguage { - other++ - } - - var status string - if expected == obtained { - status = "ok" - ok++ - } else { - status = "failed" - failed++ - if !safe { - unsafe++ - } - } - - fmt.Fprintf(w, "%s\t%s\t%s\t%v\t%s\n", filename, expected, obtained, safe, status) - - return nil - }) - - fmt.Fprintln(w) - w.Flush() - - fmt.Printf("total files: %d, ok: %d, failed: %d, unsafe: %d, other: %d\n", total, ok, failed, unsafe, other) -} - -func (s *SimpleLinguistTestSuite) TestGetLanguageByClassifier() { +func (s *SimpleLinguistTestSuite) TestGetLanguagesByClassifier() { const samples = `.linguist/samples/` test := []struct { name string filename string - candidates map[string]float64 + candidates []string expected string }{ - {name: "TestGetLanguageByClassifier_1", filename: filepath.Join(samples, "C/blob.c"), candidates: map[string]float64{"python": 1.00, "ruby": 1.00, "c": 1.00, "c++": 1.00}, expected: "C"}, - {name: "TestGetLanguageByClassifier_2", filename: filepath.Join(samples, "C/blob.c"), candidates: nil, expected: "C"}, - {name: "TestGetLanguageByClassifier_3", filename: filepath.Join(samples, "C/main.c"), candidates: nil, expected: "C"}, - {name: "TestGetLanguageByClassifier_4", filename: filepath.Join(samples, "C/blob.c"), candidates: map[string]float64{"python": 1.00, "ruby": 1.00, "c++": 1.00}, expected: "C++"}, - {name: "TestGetLanguageByClassifier_5", filename: filepath.Join(samples, "C/blob.c"), candidates: map[string]float64{"ruby": 1.00}, expected: "Ruby"}, - {name: "TestGetLanguageByClassifier_6", filename: filepath.Join(samples, "Python/django-models-base.py"), candidates: map[string]float64{"python": 1.00, "ruby": 1.00, "c": 1.00, "c++": 1.00}, expected: "Python"}, - {name: "TestGetLanguageByClassifier_7", filename: filepath.Join(samples, "Python/django-models-base.py"), candidates: nil, expected: "Python"}, + {name: "TestGetLanguagesByClassifier_1", filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"python", "ruby", "c", "c++"}, expected: "C"}, + {name: "TestGetLanguagesByClassifier_2", filename: filepath.Join(samples, "C/blob.c"), candidates: nil, expected: OtherLanguage}, + {name: "TestGetLanguagesByClassifier_3", filename: filepath.Join(samples, "C/main.c"), candidates: []string{}, expected: OtherLanguage}, + {name: "TestGetLanguagesByClassifier_4", filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"python", "ruby", "c++"}, expected: "C++"}, + {name: "TestGetLanguagesByClassifier_5", filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"ruby"}, expected: "Ruby"}, + {name: "TestGetLanguagesByClassifier_6", filename: filepath.Join(samples, "Python/django-models-base.py"), candidates: []string{"python", "ruby", "c", "c++"}, expected: "Python"}, } for _, test := range test { content, err := ioutil.ReadFile(test.filename) assert.NoError(s.T(), err) - lang := GetLanguageByClassifier(content, test.candidates, nil) - assert.Equal(s.T(), test.expected, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expected)) + languages := GetLanguagesByClassifier(test.filename, content, test.candidates) + var language string + if len(languages) == 0 { + language = OtherLanguage + } else { + language = languages[0] + } + + assert.Equal(s.T(), test.expected, language, fmt.Sprintf("%v: language = %v, expected: %v", test.name, language, test.expected)) + } +} + +func (s *SimpleLinguistTestSuite) TestGetLanguagesBySpecificClassifier() { + const samples = `.linguist/samples/` + test := []struct { + name string + filename string + candidates []string + classifier Classifier + expected string + }{ + {name: "TestGetLanguagesByClassifier_1", filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"python", "ruby", "c", "c++"}, classifier: DefaultClassifier, expected: "C"}, + {name: "TestGetLanguagesByClassifier_2", filename: filepath.Join(samples, "C/blob.c"), candidates: nil, classifier: DefaultClassifier, expected: "C"}, + {name: "TestGetLanguagesByClassifier_3", filename: filepath.Join(samples, "C/main.c"), candidates: []string{}, classifier: DefaultClassifier, expected: "C"}, + {name: "TestGetLanguagesByClassifier_4", filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"python", "ruby", "c++"}, classifier: DefaultClassifier, expected: "C++"}, + {name: "TestGetLanguagesByClassifier_5", filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"ruby"}, classifier: DefaultClassifier, expected: "Ruby"}, + {name: "TestGetLanguagesByClassifier_6", filename: filepath.Join(samples, "Python/django-models-base.py"), candidates: []string{"python", "ruby", "c", "c++"}, classifier: DefaultClassifier, expected: "Python"}, + {name: "TestGetLanguagesByClassifier_6", filename: os.DevNull, candidates: nil, classifier: DefaultClassifier, expected: OtherLanguage}, + } + + for _, test := range test { + content, err := ioutil.ReadFile(test.filename) + assert.NoError(s.T(), err) + + languages := GetLanguagesBySpecificClassifier(content, test.candidates, test.classifier) + var language string + if len(languages) == 0 { + language = OtherLanguage + } else { + language = languages[0] + } + + assert.Equal(s.T(), test.expected, language, fmt.Sprintf("%v: language = %v, expected: %v", test.name, language, test.expected)) } } @@ -349,3 +335,56 @@ func (s *SimpleLinguistTestSuite) TestGetLanguageByAlias() { assert.Equal(s.T(), test.expectedOk, ok, fmt.Sprintf("%v: ok = %v, expected: %v", test.name, ok, test.expectedOk)) } } + +func (s *SimpleLinguistTestSuite) TestLinguistCorpus() { + const ( + samplesDir = ".linguist/samples" + filenamesDir = "filenames" + ) + + var cornerCases = map[string]bool{ + "hello.ms": true, + } + + var total, failed, ok, other int + var expected string + filepath.Walk(samplesDir, func(path string, f os.FileInfo, err error) error { + if f.IsDir() { + if f.Name() != filenamesDir { + expected = f.Name() + } + + return nil + } + + filename := filepath.Base(path) + content, _ := ioutil.ReadFile(path) + + total++ + obtained := GetLanguage(filename, content) + if obtained == OtherLanguage { + obtained = "Other" + other++ + } + + var status string + if expected == obtained { + status = "ok" + ok++ + } else { + status = "failed" + failed++ + + } + + if _, ok := cornerCases[filename]; ok { + fmt.Printf("\t\t[considered corner case] %s\texpected: %s\tobtained: %s\tstatus: %s\n", filename, expected, obtained, status) + } else { + assert.Equal(s.T(), expected, obtained, fmt.Sprintf("%s\texpected: %s\tobtained: %s\tstatus: %s\n", filename, expected, obtained, status)) + } + + return nil + }) + + fmt.Printf("\t\ttotal files: %d, ok: %d, failed: %d, other: %d\n", total, ok, failed, other) +} diff --git a/content.go b/content.go index 1974528..a7eaa96 100644 --- a/content.go +++ b/content.go @@ -507,7 +507,7 @@ var ( m_Matlab_Matcher_0 = regexp.MustCompile(`(?m)^\s*%`) m_Limbo_Matcher_0 = regexp.MustCompile(`(?m)^\w+\s*:\s*module\s*{`) md_Markdown_Matcher_0 = regexp.MustCompile(`(?mi)(^[-a-z0-9=#!\*\[|>])|<\/`) - md_Markdown_Matcher_1 = regexp.MustCompile(`(?m)^$`) + md_Markdown_Matcher_1 = regexp.MustCompile(`^$`) md_GCCMachineDescription_Matcher_0 = regexp.MustCompile(`(?m)^(;;|\(define_)`) ml_OCaml_Matcher_0 = regexp.MustCompile(`(?m)(^\s*module)|let rec |match\s+(\S+\s)+with`) ml_StandardML_Matcher_0 = regexp.MustCompile(`(?m)=> |case\s+(\S+\s)+of`) diff --git a/extension.go b/extension.go index 455ec22..247ce12 100644 --- a/extension.go +++ b/extension.go @@ -28,9 +28,6 @@ var languagesByExtension = map[string][]string{ ".8xp": {"TI Program"}, ".8xp.txt": {"TI Program"}, ".9": {"Roff"}, - ".E": {"E"}, - ".JSON-tmLanguage": {"JSON"}, - ".ML": {"Standard ML"}, "._coffee": {"CoffeeScript"}, "._js": {"JavaScript"}, "._ls": {"LiveScript"}, @@ -228,7 +225,7 @@ var languagesByExtension = map[string][]string{ ".dyalog": {"APL"}, ".dyl": {"Dylan"}, ".dylan": {"Dylan"}, - ".e": {"Eiffel"}, + ".e": {"E", "Eiffel"}, ".eam.fs": {"Formatted"}, ".ebnf": {"EBNF"}, ".ebuild": {"Gentoo Ebuild"}, @@ -415,6 +412,7 @@ var languagesByExtension = map[string][]string{ ".jsfl": {"JavaScript"}, ".jsm": {"JavaScript"}, ".json": {"JSON"}, + ".json-tmlanguage": {"JSON"}, ".json5": {"JSON5"}, ".jsonld": {"JSONLD"}, ".jsp": {"Java Server Pages"}, @@ -508,7 +506,7 @@ var languagesByExtension = map[string][]string{ ".mkii": {"TeX"}, ".mkiv": {"TeX"}, ".mkvi": {"TeX"}, - ".ml": {"OCaml"}, + ".ml": {"OCaml", "Standard ML"}, ".ml4": {"OCaml"}, ".mli": {"OCaml"}, ".mll": {"OCaml"}, @@ -807,11 +805,11 @@ var languagesByExtension = map[string][]string{ ".ss": {"Scheme"}, ".ssjs": {"JavaScript"}, ".st": {"HTML", "Smalltalk"}, - ".stTheme": {"XML"}, ".stan": {"Stan"}, ".sthlp": {"Stata"}, ".ston": {"STON"}, ".storyboard": {"XML"}, + ".sttheme": {"XML"}, ".sty": {"TeX"}, ".styl": {"Stylus"}, ".sublime-build": {"Sublime Text Config"}, @@ -853,13 +851,13 @@ var languagesByExtension = map[string][]string{ ".tl": {"Type Language"}, ".tla": {"TLA"}, ".tm": {"Tcl"}, - ".tmCommand": {"XML"}, - ".tmLanguage": {"XML"}, - ".tmPreferences": {"XML"}, - ".tmSnippet": {"XML"}, - ".tmTheme": {"XML"}, ".tmac": {"Roff"}, + ".tmcommand": {"XML"}, ".tml": {"XML"}, + ".tmlanguage": {"XML"}, + ".tmpreferences": {"XML"}, + ".tmsnippet": {"XML"}, + ".tmtheme": {"XML"}, ".tmux": {"Shell"}, ".toc": {"TeX", "World of Warcraft Addon Data"}, ".toml": {"TOML"}, @@ -1099,7 +1097,7 @@ var extensionsByLanguage = map[string][]string{ "Dockerfile": {".dockerfile"}, "Dogescript": {".djs"}, "Dylan": {".dylan", ".dyl", ".intr", ".lid"}, - "E": {".E"}, + "E": {".e"}, "EBNF": {".ebnf"}, "ECL": {".ecl", ".eclxml"}, "ECLiPSe": {".ecl"}, @@ -1182,7 +1180,7 @@ var extensionsByLanguage = map[string][]string{ "Isabelle": {".thy"}, "J": {".ijs"}, "JFlex": {".flex", ".jflex"}, - "JSON": {".json", ".geojson", ".JSON-tmLanguage", ".topojson"}, + "JSON": {".json", ".geojson", ".json-tmlanguage", ".topojson"}, "JSON5": {".json5"}, "JSONLD": {".jsonld"}, "JSONiq": {".jq"}, @@ -1379,7 +1377,7 @@ var extensionsByLanguage = map[string][]string{ "Spline Font Database": {".sfd"}, "Squirrel": {".nut"}, "Stan": {".stan"}, - "Standard ML": {".ML", ".fun", ".sig", ".sml"}, + "Standard ML": {".ml", ".fun", ".sig", ".sml"}, "Stata": {".do", ".ado", ".doh", ".ihlp", ".mata", ".matah", ".sthlp"}, "Stylus": {".styl"}, "SubRip Text": {".srt"}, @@ -1426,7 +1424,7 @@ var extensionsByLanguage = map[string][]string{ "World of Warcraft Addon Data": {".toc"}, "X10": {".x10"}, "XC": {".xc"}, - "XML": {".xml", ".adml", ".admx", ".ant", ".axml", ".builds", ".ccxml", ".clixml", ".cproject", ".csl", ".csproj", ".ct", ".dita", ".ditamap", ".ditaval", ".dll.config", ".dotsettings", ".filters", ".fsproj", ".fxml", ".glade", ".gml", ".grxml", ".iml", ".ivy", ".jelly", ".jsproj", ".kml", ".launch", ".mdpolicy", ".mjml", ".mm", ".mod", ".mxml", ".nproj", ".nuspec", ".odd", ".osm", ".pkgproj", ".plist", ".pluginspec", ".props", ".ps1xml", ".psc1", ".pt", ".rdf", ".resx", ".rss", ".sch", ".scxml", ".sfproj", ".srdf", ".storyboard", ".stTheme", ".sublime-snippet", ".targets", ".tmCommand", ".tml", ".tmLanguage", ".tmPreferences", ".tmSnippet", ".tmTheme", ".ts", ".tsx", ".ui", ".urdf", ".ux", ".vbproj", ".vcxproj", ".vsixmanifest", ".vssettings", ".vstemplate", ".vxml", ".wixproj", ".wsdl", ".wsf", ".wxi", ".wxl", ".wxs", ".x3d", ".xacro", ".xaml", ".xib", ".xlf", ".xliff", ".xmi", ".xml.dist", ".xproj", ".xsd", ".xul", ".zcml"}, + "XML": {".xml", ".adml", ".admx", ".ant", ".axml", ".builds", ".ccxml", ".clixml", ".cproject", ".csl", ".csproj", ".ct", ".dita", ".ditamap", ".ditaval", ".dll.config", ".dotsettings", ".filters", ".fsproj", ".fxml", ".glade", ".gml", ".grxml", ".iml", ".ivy", ".jelly", ".jsproj", ".kml", ".launch", ".mdpolicy", ".mjml", ".mm", ".mod", ".mxml", ".nproj", ".nuspec", ".odd", ".osm", ".pkgproj", ".plist", ".pluginspec", ".props", ".ps1xml", ".psc1", ".pt", ".rdf", ".resx", ".rss", ".sch", ".scxml", ".sfproj", ".srdf", ".storyboard", ".sttheme", ".sublime-snippet", ".targets", ".tmcommand", ".tml", ".tmlanguage", ".tmpreferences", ".tmsnippet", ".tmtheme", ".ts", ".tsx", ".ui", ".urdf", ".ux", ".vbproj", ".vcxproj", ".vsixmanifest", ".vssettings", ".vstemplate", ".vxml", ".wixproj", ".wsdl", ".wsf", ".wxi", ".wxl", ".wxs", ".x3d", ".xacro", ".xaml", ".xib", ".xlf", ".xliff", ".xmi", ".xml.dist", ".xproj", ".xsd", ".xul", ".zcml"}, "XPages": {".xsp-config", ".xsp.metadata"}, "XProc": {".xpl", ".xproc"}, "XQuery": {".xquery", ".xq", ".xql", ".xqm", ".xqy"}, diff --git a/filename.go b/filename.go index ac88912..b4f19af 100644 --- a/filename.go +++ b/filename.go @@ -16,22 +16,39 @@ var languagesByFilename = map[string][]string{ ".bashrc": {"Shell"}, ".clang-format": {"YAML"}, ".classpath": {"XML"}, + ".cproject": {"XML"}, + ".cshrc": {"Shell"}, + ".editorconfig": {"INI"}, ".emacs": {"Emacs Lisp"}, ".emacs.desktop": {"Emacs Lisp"}, ".factor-boot-rc": {"Factor"}, ".factor-rc": {"Factor"}, ".gclient": {"Python"}, + ".gemrc": {"YAML"}, + ".gitconfig": {"INI"}, + ".gn": {"GN"}, ".gnus": {"Emacs Lisp"}, + ".gvimrc": {"Vim script"}, + ".htaccess": {"ApacheConf"}, ".irbrc": {"Ruby"}, ".jshintrc": {"JSON"}, + ".login": {"Shell"}, ".nvimrc": {"Vim script"}, + ".php": {"PHP"}, ".php_cs": {"PHP"}, ".php_cs.dist": {"PHP"}, + ".profile": {"Shell"}, ".project": {"XML"}, ".pryrc": {"Ruby"}, ".spacemacs": {"Emacs Lisp"}, ".vimrc": {"Vim script"}, ".viper": {"Emacs Lisp"}, + ".zlogin": {"Shell"}, + ".zlogout": {"Shell"}, + ".zprofile": {"Shell"}, + ".zshenv": {"Shell"}, + ".zshrc": {"Shell"}, + "9fs": {"Shell"}, "APKBUILD": {"Alpine Abuild"}, "App.config": {"XML"}, "Appraisals": {"Ruby"}, @@ -43,8 +60,10 @@ var languagesByFilename = map[string][]string{ "Buildfile": {"Ruby"}, "CMakeLists.txt": {"CMake"}, "COPYING": {"Text"}, + "COPYING.regex": {"Text"}, "COPYRIGHT.regex": {"Text"}, "Cakefile": {"CoffeeScript"}, + "Capfile": {"Ruby"}, "Cask": {"Emacs Lisp"}, "Dangerfile": {"Ruby"}, "Deliverfile": {"Ruby"}, @@ -105,21 +124,31 @@ var languagesByFilename = map[string][]string{ "_emacs": {"Emacs Lisp"}, "_vimrc": {"Vim script"}, "abbrev_defs": {"Emacs Lisp"}, + "ack": {"Perl"}, "ant.xml": {"Ant Build System"}, + "apache2.conf": {"ApacheConf"}, + "bash_logout": {"Shell"}, + "bash_profile": {"Shell"}, + "bashrc": {"Shell"}, "build.xml": {"Ant Build System"}, "buildfile": {"Ruby"}, "click.me": {"Text"}, "composer.lock": {"JSON"}, "configure.ac": {"M4Sugar"}, + "cshrc": {"Shell"}, "delete.me": {"Text"}, "descrip.mmk": {"Module Management System"}, "descrip.mms": {"Module Management System"}, + "expr-dist": {"R"}, "gradlew": {"Shell"}, "gvimrc": {"Vim script"}, + "httpd.conf": {"ApacheConf"}, "keep.me": {"Text"}, "ld.script": {"Linker Script"}, + "login": {"Shell"}, "makefile": {"Makefile"}, "makefile.sco": {"Makefile"}, + "man": {"Shell"}, "mcmod.info": {"JSON"}, "meson.build": {"Meson"}, "meson_options.txt": {"Meson"}, @@ -129,15 +158,25 @@ var languagesByFilename = map[string][]string{ "mmt": {"Roff"}, "nginx.conf": {"Nginx"}, "nvimrc": {"Vim script"}, + "owh": {"Tcl"}, "packages.config": {"XML"}, "pom.xml": {"Maven POM"}, + "profile": {"Shell"}, "read.me": {"Text"}, + "readme.1st": {"Text"}, "rebar.config": {"Erlang"}, "rebar.config.lock": {"Erlang"}, "rebar.lock": {"Erlang"}, "riemann.config": {"Clojure"}, + "script": {"C"}, + "starfield": {"Tcl"}, "test.me": {"Text"}, "vimrc": {"Vim script"}, "wscript": {"Python"}, "xcompose": {"XCompose"}, + "zlogin": {"Shell"}, + "zlogout": {"Shell"}, + "zprofile": {"Shell"}, + "zshenv": {"Shell"}, + "zshrc": {"Shell"}, } diff --git a/frequencies.go b/frequencies.go index dd9601a..c9dccb1 100644 --- a/frequencies.go +++ b/frequencies.go @@ -386,557 +386,666 @@ var DefaultClassifier Classifier = &classifier{ }, tokensLogProbabilities: map[string]map[string]float64{ "1C Enterprise": map[string]float64{ - "&": -6.672223, - "(": -4.054827, - ")": -4.054827, - "*": -9.667955, - "+": -6.202219, - ",": -5.082988, - "-": -9.667955, - ".": -3.954222, - "/": -9.667955, - ";": -3.874941, - "<": -6.777583, - "=": -4.474998, - ">": -6.723516, - "?": -8.974808, - "DOM": -8.974808, - "DOM.": -9.667955, - "HTML": -8.974808, - "HTML.": -8.974808, - "Null": -9.667955, - "XML": -7.270060, - "XML.": -7.722045, - "_": -7.470730, - "А": -8.281661, - "Б": -8.058517, - "В": -4.698142, - "Г": -8.281661, - "Д": -4.698142, - "Е": -5.114078, - "Ж": -9.667955, - "З": -4.586551, - "И": -5.311246, - "К": -4.311369, - "Л": -6.233968, - "М": -6.266758, - "Н": -4.586551, - "О": -4.618099, - "П": -3.984375, - "Р": -5.839314, - "С": -4.431513, - "Т": -4.400097, - "У": -6.004393, - "Ф": -5.979076, - "Х": -8.974808, - "Ц": -5.607512, - "Ч": -6.576913, - "Ш": -6.623433, - "Э": -5.796754, - "а": -2.500146, - "б": -4.274328, - "в": -3.881058, - "г": -4.808143, - "д": -4.036743, - "е": -2.538658, - "ж": -5.146166, - "з": -4.839641, - "и": -2.864450, - "й": -4.684348, - "к": -3.479691, - "л": -3.220649, - "м": -4.076968, - "н": -2.950150, - "о": -2.582054, - "п": -4.301979, - "р": -3.197156, - "с": -3.578910, - "т": -2.868899, - "у": -4.029600, - "ф": -5.817807, - "х": -6.202219, - "ц": -4.306663, - "ч": -4.657320, - "ш": -7.470730, - "щ": -5.524820, - "ъ": -5.979076, - "ы": -3.793024, - "ь": -4.043938, - "ю": -6.112607, - "я": -4.562010, - "\ufeff": -8.974808, + "#": -8.978471, + "&": -6.675886, + "(": -4.065816, + ")": -4.065816, + "*": -9.671619, + "+": -6.205883, + ",": -5.086651, + "-": -8.978471, + ".": -3.957886, + "/": -9.671619, + "//": -8.573006, + "//////////////////////////////////////////////////////////////////////////////": -8.978471, + ";": -3.878605, + "<": -6.781247, + "=": -4.478662, + ">": -6.727180, + "?": -8.978471, + "DOM": -8.978471, + "DOM.": -9.671619, + "HTML": -8.978471, + "HTML.": -8.978471, + "Null": -9.671619, + "XML": -7.273723, + "XML.": -7.725708, + "_": -7.474394, + "А": -8.285324, + "Б": -8.062181, + "В": -4.701805, + "Г": -8.285324, + "Д": -4.722859, + "Е": -5.107270, + "Ж": -9.671619, + "З": -4.584022, + "И": -5.289592, + "К": -4.310326, + "Л": -6.237631, + "М": -6.270421, + "Н": -4.590214, + "О": -4.641181, + "П": -4.001738, + "Р": -5.842977, + "С": -4.435177, + "Т": -4.403760, + "У": -6.060701, + "Ф": -5.982739, + "Х": -8.978471, + "Ц": -5.611176, + "Ч": -6.580576, + "Ш": -6.627096, + "Э": -5.821471, + "а": -2.506125, + "б": -4.291721, + "в": -3.860478, + "г": -4.804084, + "д": -4.036829, + "е": -2.541520, + "ж": -5.139019, + "з": -4.827431, + "и": -2.859273, + "й": -4.688012, + "к": -3.479256, + "л": -3.214849, + "м": -4.073197, + "н": -2.957448, + "о": -2.573243, + "п": -4.273456, + "р": -3.199272, + "с": -3.569060, + "т": -2.863684, + "у": -4.029711, + "ф": -5.821471, + "х": -6.237631, + "ц": -4.319760, + "ч": -4.654339, + "ш": -7.369033, + "щ": -5.528484, + "ъ": -5.982739, + "ы": -3.805150, + "ь": -4.029711, + "ю": -6.116270, + "я": -4.565673, + "\ufeff": -8.978471, }, "ABAP": map[string]float64{ - "!": -3.263996, - "(": -4.442651, - ")": -4.442651, - "*": -2.427748, - "+": -4.219508, - "-": -4.219508, - ".": -3.631721, - "<-()]>": -5.135798, - "": -5.135798, - "": -4.442651, - "": -4.730333, - "=": -3.526361, - ">": -4.442651, - "CLASS": -5.135798, - "CL_CSV_PARSER": -4.219508, - "CONSTRUCTOR": -5.828946, - "CSV": -5.828946, - "CX": -5.828946, - "CX_CSV_PARSE_ERROR": -5.828946, - "DEFINITION": -5.135798, - "Get": -5.828946, - "IMPLEMENTATION": -5.828946, - "Instance": -5.135798, - "Method": -5.135798, - "Parse": -5.828946, - "Private": -5.828946, - "Public": -5.828946, - "RETURNING": -5.828946, - "SIGNATURE": -5.135798, - "STRINGTAB": -4.730333, - "Space": -5.135798, - "TYPE": -4.730333, - "This": -5.828946, - "[": -4.219508, - "_LINES": -5.828946, - "_csvstring": -5.135798, - "_delegate": -5.828946, - "_lines": -5.828946, - "_lines.": -5.828946, - "_parse_line": -5.135798, - "_separator": -5.828946, - "_skip_first_line": -5.828946, - "_textindicator": -5.828946, - "`": -4.442651, - "abap": -5.828946, - "abap_bool": -5.135798, - "abap_true.": -5.135798, - "an": -5.828946, - "append": -5.135798, - "assigning": -5.828946, - "at": -5.135798, - "c": -5.135798, - "char": -5.135798, - "cl_abap_char_utilities": -5.828946, - "cl_csv_parser": -5.135798, - "cl_object": -5.828946, - "class": -5.135798, - "clear": -5.828946, - "concatenate": -4.442651, - "constants": -5.828946, - "constructor": -5.135798, - "constructor.": -5.828946, - "cr_lf": -5.828946, - "create": -5.828946, - "csv": -5.828946, - "csvstring": -5.828946, - "csvstring.": -5.828946, - "csvvalue": -4.037186, - "csvvalue.": -4.219508, - "csvvalues.": -5.135798, - "cx_csv_parse_error": -5.135798, - "data": -4.730333, - "definition": -5.828946, - "delegate": -5.828946, - "delegate.": -5.828946, - "do": -4.730333, - "e": -5.828946, - "else.": -4.442651, - "endclass.": -5.828946, - "endif.": -4.037186, - "endmethod.": -5.135798, - "endwhile.": -5.135798, - "error": -5.828946, - "exception": -5.828946, - "exporting": -5.828946, - "field": -5.828946, - "files": -4.730333, - "final": -5.828946, - "formatting": -5.828946, - "from": -5.828946, - "here": -4.730333, - "if_csv_parser_delegate": -5.828946, - "implementation.": -5.828946, - "importing": -5.828946, - "in": -5.828946, - "include": -4.730333, - "indicates": -5.828946, - "inheriting": -5.828946, - "into": -4.037186, - "is_first_line": -5.828946, - "line": -5.828946, - "lines": -4.442651, - "loop": -5.828946, - "message": -5.135798, - "method": -5.135798, - "methods": -5.135798, - "msg.": -5.135798, - "not": -4.730333, - "of": -5.135798, - "other": -4.730333, - "parse": -5.135798, - "pools": -5.828946, - "pos": -5.135798, - "private": -5.828946, - "protected": -5.828946, - "public": -4.730333, - "raise": -5.828946, - "raising": -5.828946, - "ref": -5.828946, - "returning.": -5.828946, - "section.": -4.730333, - "separator": -5.828946, - "separator.": -5.828946, - "skip_first_line": -5.828946, - "skip_first_line.": -5.828946, - "source": -4.730333, - "split": -5.828946, - "standard": -5.135798, - "string": -5.828946, - "string.": -4.730333, - "super": -5.828946, - "symbols": -5.828946, - "table": -4.730333, - "text_ended": -5.828946, - "the": -4.730333, - "to": -4.730333, - "type": -3.431050, - "value": -5.135798, - "values": -5.135798, - "|": -3.883035, + "!": -4.637712, + "\"": -7.202661, + "(": -5.816367, + ")": -5.816367, + "*": -3.619142, + "+": -5.005437, + "-": -0.322277, + ".": -5.005437, + "<-()]>": -6.509514, + "": -6.509514, + "": -5.816367, + "": -6.104049, + "=": -4.900076, + ">": -5.123220, + "ABAP_BOOL": -7.202661, + "C": -7.202661, + "CLASS": -6.509514, + "CL_CSV_PARSER": -5.410902, + "CONSTRUCTOR": -7.202661, + "CSV": -7.202661, + "CSVSTRING": -7.202661, + "CX": -7.202661, + "CX_CSV_PARSE_ERROR": -7.202661, + "DEFINITION": -6.509514, + "DELEGATE": -7.202661, + "Get": -7.202661, + "IF_CSV_PARSER_DELEGATE": -7.202661, + "IMPLEMENTATION": -6.509514, + "Instance": -6.509514, + "Method": -6.509514, + "Parse": -7.202661, + "Private": -7.202661, + "Public": -7.202661, + "REF": -7.202661, + "RETURNING": -7.202661, + "SEPARATOR": -7.202661, + "SIGNATURE": -6.509514, + "SKIP_FIRST_LINE": -7.202661, + "STRING": -7.202661, + "STRINGTAB": -6.104049, + "Space": -6.509514, + "TO": -7.202661, + "TYPE": -5.256751, + "This": -7.202661, + "[": -5.593223, + "]": -5.816367, + "_LINES": -7.202661, + "_csvstring": -6.509514, + "_delegate": -7.202661, + "_lines": -7.202661, + "_lines.": -7.202661, + "_parse_line": -6.509514, + "_separator": -7.202661, + "_skip_first_line": -7.202661, + "_textindicator": -7.202661, + "`": -5.816367, + "abap": -7.202661, + "abap_bool": -6.509514, + "abap_true.": -6.509514, + "an": -7.202661, + "append": -6.509514, + "assigning": -7.202661, + "at": -6.509514, + "c": -6.509514, + "char": -6.509514, + "cl_abap_char_utilities": -7.202661, + "cl_csv_parser": -6.509514, + "cl_object": -7.202661, + "class": -6.509514, + "clear": -7.202661, + "concatenate": -5.816367, + "constants": -7.202661, + "constructor": -6.509514, + "constructor.": -7.202661, + "cr_lf": -7.202661, + "create": -7.202661, + "csv": -7.202661, + "csvstring": -7.202661, + "csvstring.": -7.202661, + "csvvalue": -5.410902, + "csvvalue.": -5.593223, + "csvvalues.": -6.509514, + "cx_csv_parse_error": -6.509514, + "data": -6.104049, + "definition": -7.202661, + "delegate": -7.202661, + "delegate.": -7.202661, + "do": -6.104049, + "e": -7.202661, + "else.": -5.816367, + "endclass.": -7.202661, + "endif.": -5.410902, + "endmethod.": -6.509514, + "endwhile.": -6.509514, + "error": -7.202661, + "exception": -7.202661, + "exporting": -7.202661, + "field": -7.202661, + "files": -6.104049, + "final": -7.202661, + "formatting": -7.202661, + "from": -7.202661, + "here": -6.104049, + "if_csv_parser_delegate": -7.202661, + "implementation.": -7.202661, + "importing": -7.202661, + "in": -7.202661, + "include": -6.104049, + "indicates": -7.202661, + "inheriting": -7.202661, + "into": -5.410902, + "is_first_line": -7.202661, + "line": -7.202661, + "lines": -5.816367, + "loop": -7.202661, + "message": -6.509514, + "method": -6.509514, + "methods": -6.509514, + "msg.": -6.509514, + "not": -6.104049, + "of": -6.509514, + "other": -6.104049, + "parse": -6.509514, + "pools": -7.202661, + "pos": -6.509514, + "private": -7.202661, + "protected": -7.202661, + "public": -6.104049, + "raise": -7.202661, + "raising": -7.202661, + "ref": -7.202661, + "returning.": -7.202661, + "section.": -6.104049, + "separator": -7.202661, + "separator.": -7.202661, + "skip_first_line": -7.202661, + "skip_first_line.": -7.202661, + "source": -6.104049, + "split": -7.202661, + "standard": -6.509514, + "string": -7.202661, + "string.": -6.104049, + "super": -7.202661, + "symbols": -7.202661, + "table": -6.104049, + "text_ended": -7.202661, + "the": -6.104049, + "to": -6.104049, + "type": -4.804766, + "value": -6.509514, + "values": -6.509514, + "|": -5.256751, }, "ABNF": map[string]float64{ - "(": -3.938340, - ")": -4.056123, - "*": -4.631487, - "*DIGIT": -6.828712, - "*basic": -6.135565, - "*literal": -6.828712, - "*newline": -6.828712, - "*non": -6.828712, - ",": -5.036953, - "-": -1.576439, - "/": -3.165150, - ":": -5.442418, - ";": -2.821379, - "=": -2.434263, - "ABNF": -6.135565, - "ALPHA": -6.135565, - "Array": -6.135565, - "Basic": -5.730100, - "Boolean": -6.828712, - "Built": -6.828712, - "Comment": -6.828712, - "DIGIT": -4.120662, - "Datetime": -6.828712, - "Float": -6.828712, - "HEXDIG": -6.828712, - "Inline": -6.828712, - "Integer": -6.828712, - "Key": -6.828712, - "License": -6.828712, - "Literal": -6.135565, - "MIT": -6.828712, - "Multiline": -6.135565, - "Newline": -6.828712, - "RFC": -6.135565, - "See": -6.828712, - "Source": -6.828712, - "Standard": -6.828712, - "String": -5.219274, - "Strings": -6.828712, - "TOML": -6.135565, - "Table": -5.442418, - "This": -6.828712, - "Value": -6.828712, - "Whitespace": -6.828712, - "[": -4.631487, - "]": -4.631487, - "according": -6.828712, - "an": -6.828712, - "apostraphe": -5.036953, - "array": -3.938340, - "as": -6.828712, - "attempt": -6.828712, - "based": -6.135565, - "basic": -4.189655, - "body": -5.442418, - "boolean": -6.135565, - "char": -4.749271, - "clarity": -6.828712, - "close": -4.749271, - "comment": -4.749271, - "date": -4.526127, - "decimal": -6.135565, - "define": -6.828712, - "defined": -6.135565, - "delim": -5.036953, - "digit": -6.135565, - "e": -6.135565, - "empty": -5.730100, - "eol": -6.135565, - "escape": -6.135565, - "escaped": -6.828712, - "exp": -5.730100, - "expression": -5.730100, - "false": -6.135565, - "float": -6.135565, - "for": -6.828712, - "frac": -5.730100, - "full": -5.442418, - "fullyear": -6.135565, - "grammar": -6.828712, - "here": -6.828712, - "hour": -5.730100, - "http": -6.828712, - "https": -6.828712, - "in": -5.442418, - "inline": -4.263763, - "int": -5.442418, - "integer": -5.442418, - "is": -6.828712, - "key": -4.120662, - "keyval": -5.036953, - "keyvals": -5.219274, - "leap": -6.828712, - "literal": -4.343805, - "mark": -4.749271, - "mday": -6.135565, - "minus": -6.135565, - "minute": -5.730100, - "ml": -3.832980, - "month": -6.135565, - "month/year": -6.828712, - "newline": -5.219274, - "newlines": -5.219274, - "non": -5.442418, - "numoffset": -6.135565, - "offset": -6.135565, - "on": -6.135565, - "open": -4.749271, - "pairs": -6.828712, - "partial": -6.135565, - "plus": -6.135565, - "point": -6.135565, - "prefixable": -6.135565, - "quotation": -4.749271, - "quoted": -6.135565, - "reproduced": -6.828712, - "rules": -6.828712, - "secfrac": -6.135565, - "second": -5.730100, - "sep": -4.343805, - "start": -6.135565, - "std": -5.036953, - "string": -4.056123, - "symbol": -6.135565, - "table": -3.427515, - "terms": -6.828712, - "the": -6.828712, - "time": -3.832980, - "to": -6.135565, - "toml": -6.828712, - "true": -6.135565, - "underscore": -5.730100, - "unescaped": -5.730100, - "unquoted": -6.135565, - "val": -5.036953, - "values": -5.730100, - "ws": -4.120662, - "zero": -6.135565, + "%": -3.171745, + "'": -7.142037, + "(": -4.251665, + ")": -4.197598, + "*": -4.944812, + "*DIGIT": -7.142037, + "*basic": -6.448889, + "*literal": -7.142037, + "*newline": -7.142037, + "*non": -7.142037, + "+": -4.744141, + ",": -5.062595, + "-": -1.785450, + ".": -6.043424, + "/": -2.983153, + "//github.com/toml": -7.142037, + "//www.ietf.org/rfc/rfc": -7.142037, + ":": -5.755742, + ";": -2.588160, + "=": -2.747587, + "A": -5.196126, + "ABNF": -6.448889, + "ALPHA": -6.448889, + "Apostrophe": -7.142037, + "Array": -6.448889, + "B": -5.196126, + "B.": -7.142037, + "Basic": -6.043424, + "Boolean": -7.142037, + "Built": -7.142037, + "C": -5.350277, + "C.": -7.142037, + "CRLF": -7.142037, + "Comma": -6.448889, + "Comment": -7.142037, + "D": -4.839451, + "D.": -6.448889, + "DIGIT": -4.433986, + "Datetime": -7.142037, + "Double": -6.448889, + "E": -5.755742, + "Float": -7.142037, + "HEXDIG": -6.043424, + "Horizontal": -7.142037, + "Inline": -7.142037, + "Integer": -7.142037, + "Key": -7.142037, + "LF": -7.142037, + "Left": -7.142037, + "License": -7.142037, + "Literal": -6.448889, + "MIT": -7.142037, + "Multiline": -6.448889, + "Newline": -7.142037, + "Period": -7.142037, + "RFC": -6.448889, + "Right": -7.142037, + "See": -7.142037, + "Source": -7.142037, + "Space": -7.142037, + "Standard": -7.142037, + "String": -5.532599, + "Strings": -7.142037, + "TOML": -6.448889, + "Table": -5.755742, + "This": -7.142037, + "U": -4.839451, + "UXXXXXXXX": -7.142037, + "Value": -7.142037, + "Whitespace": -7.142037, + "XXXX": -7.142037, + "XXXXXXXX": -7.142037, + "Z": -6.448889, + "[": -4.577087, + "\\": -6.448889, + "]": -4.577087, + "_": -6.448889, + "a": -6.448889, + "according": -7.142037, + "an": -7.142037, + "apostraphe": -5.350277, + "array": -4.251665, + "as": -7.142037, + "attempt": -7.142037, + "b": -7.142037, + "backspace": -7.142037, + "based": -6.448889, + "basic": -4.502979, + "body": -5.755742, + "boolean": -6.448889, + "bracket": -5.755742, + "carriage": -7.142037, + "char": -5.062595, + "clarity": -7.142037, + "close": -5.062595, + "comment": -5.062595, + "date": -4.839451, + "decimal": -6.448889, + "define": -7.142037, + "defined": -6.448889, + "delim": -5.350277, + "digit": -6.448889, + "e": -6.043424, + "empty": -6.043424, + "eol": -7.142037, + "escape": -6.448889, + "escaped": -7.142037, + "exp": -6.043424, + "expression": -6.043424, + "f": -7.142037, + "false": -6.043424, + "feed": -6.448889, + "float": -6.448889, + "for": -7.142037, + "form": -7.142037, + "frac": -6.043424, + "full": -5.755742, + "fullyear": -6.448889, + "grammar": -7.142037, + "here": -7.142037, + "hour": -6.043424, + "http": -7.142037, + "https": -7.142037, + "in": -5.755742, + "inline": -4.577087, + "int": -5.755742, + "integer": -5.755742, + "is": -7.142037, + "key": -4.433986, + "keyval": -5.350277, + "keyvals": -5.532599, + "lang/toml": -7.142037, + "leap": -7.142037, + "left": -7.142037, + "line": -7.142037, + "literal": -4.657130, + "mark": -4.944812, + "mday": -6.448889, + "minus": -6.448889, + "minute": -6.043424, + "ml": -4.146304, + "month": -6.448889, + "month/year": -7.142037, + "n": -7.142037, + "newline": -5.532599, + "newlines": -5.532599, + "non": -6.043424, + "numoffset": -6.448889, + "offset": -6.448889, + "on": -6.448889, + "open": -5.062595, + "pairs": -7.142037, + "partial": -6.448889, + "plus": -6.448889, + "point": -6.448889, + "prefixable": -6.448889, + "quotation": -4.944812, + "quoted": -6.448889, + "r": -7.142037, + "reproduced": -7.142037, + "return": -7.142037, + "reverse": -7.142037, + "right": -7.142037, + "rules": -7.142037, + "secfrac": -6.448889, + "second": -6.043424, + "sep": -4.657130, + "solidus": -6.448889, + "square": -5.755742, + "start": -6.448889, + "std": -5.350277, + "string": -4.369448, + "symbol": -6.448889, + "t": -7.142037, + "tab": -6.448889, + "table": -3.740839, + "terms": -7.142037, + "the": -7.142037, + "time": -4.146304, + "to": -6.448889, + "toml": -7.142037, + "true": -6.043424, + "txt": -7.142037, + "uXXXX": -7.142037, + "underscore": -6.043424, + "unescaped": -6.043424, + "unquoted": -6.448889, + "val": -5.350277, + "values": -6.043424, + "ws": -4.006542, + "x": -3.171745, + "z": -6.448889, + "zero": -6.448889, + "{": -7.142037, + "}": -7.142037, }, "AGS Script": map[string]float64{ - "!": -5.422745, - "&&": -5.710427, - "(": -2.184066, - ")": -2.184066, - "*control": -4.283311, - "*theGui": -7.502186, - "+": -5.556276, - ",": -3.610366, - "-": -5.017280, - ":": -6.809039, - ";": -2.354692, - "<": -7.502186, - "=": -2.169468, - ">": -6.809039, - "DISTANCE": -4.324133, - "Debug": -6.115892, - "DeleteSaveSlot": -7.502186, - "Display": -6.115892, - "EventType": -7.502186, - "GUI": -7.502186, - "GUIControl": -4.283311, - "IsGamePaused": -6.403574, - "IsInterfaceEnabled": -6.809039, - "IsKeyPressed": -4.668973, - "IsSpeechVoxAvailable": -7.502186, - "KeyboardMovement": -6.809039, - "KeyboardMovement_CurrentDirection": -5.556276, - "KeyboardMovement_Directions": -6.115892, - "KeyboardMovement_KeyDown": -5.892749, - "KeyboardMovement_KeyDownLeft": -6.403574, - "KeyboardMovement_KeyDownRight": -6.403574, - "KeyboardMovement_KeyLeft": -5.892749, - "KeyboardMovement_KeyRight": -5.892749, - "KeyboardMovement_KeyStop": -6.403574, - "KeyboardMovement_KeyUp": -5.892749, - "KeyboardMovement_KeyUpLeft": -6.403574, - "KeyboardMovement_KeyUpRight": -6.403574, - "KeyboardMovement_Mode": -6.115892, - "KeyboardMovement_Modes": -6.115892, - "MouseButton": -4.411144, - "ProcessClick": -7.502186, - "QuitGame": -6.809039, - "RestoreGameSlot": -7.502186, - "SaveGameSlot": -7.502186, - "SetMode": -6.809039, - "SetVoiceMode": -6.115892, - "System.SupportsGammaControl": -7.502186, - "System.Volume": -7.502186, - "Wait": -6.403574, - "[": -5.710427, - "]": -5.710427, - "btnAbout_OnClick": -7.502186, - "btnCancelRestore_OnClick": -7.502186, - "btnCancelSave_OnClick": -7.502186, - "btnDeleteSave_OnClick": -7.502186, - "btnIconAbout_Click": -7.502186, - "btnIconCurInv_Click": -7.502186, - "btnIconExit_Click": -7.502186, - "btnIconInv_Click": -7.502186, - "btnIconLoad": -7.502186, - "btnIconLoad_Click": -6.809039, - "btnIconSave": -7.502186, - "btnIconSave_Click": -6.809039, - "btnInvDown_Click": -7.502186, - "btnInvOK_Click": -7.502186, - "btnInvSelect_Click": -7.502186, - "btnInvUp_Click": -7.502186, - "btnLoad_OnClick": -7.502186, - "btnQuit_OnClick": -7.502186, - "btnRestoreGame_OnClick": -7.502186, - "btnResume_OnClick": -7.502186, - "btnSaveGame_OnClick": -6.809039, - "btnSave_OnClick": -7.502186, - "btnVoice.Text": -5.556276, - "btnVoice.Visible": -7.502186, - "button": -4.324133, - "cEgo_Interact": -7.502186, - "cEgo_Look": -7.502186, - "cEgo_Talk": -7.502186, - "close_restore_game_dialog": -6.403574, - "close_save_game_dialog": -6.403574, - "control": -7.502186, - "data": -7.502186, - "dx": -4.506454, - "dy": -4.506454, - "eEventLeaveRoom": -7.502186, - "eKeyCtrlA": -7.502186, - "eKeyCtrlS": -7.502186, - "eKeyCtrlV": -7.502186, - "eKeyCtrlW": -7.502186, - "eKeyCtrlX": -7.502186, - "eKeyEscape": -7.502186, - "eKeyboardMovement_Down": -5.892749, - "eKeyboardMovement_DownLeft": -5.892749, - "eKeyboardMovement_DownRight": -5.892749, - "eKeyboardMovement_Left": -5.892749, - "eKeyboardMovement_None": -6.809039, - "eKeyboardMovement_Pressing": -6.809039, - "eKeyboardMovement_Right": -5.892749, - "eKeyboardMovement_Stop": -5.304962, - "eKeyboardMovement_Tapping": -6.809039, - "eKeyboardMovement_Up": -5.892749, - "eKeyboardMovement_UpLeft": -5.892749, - "eKeyboardMovement_UpRight": -5.892749, - "eModeInteract": -7.502186, - "eModePointer": -5.892749, - "eModeTalkto": -7.502186, - "eModeUseinv": -6.809039, - "eModeWalkto": -7.502186, - "eMouseLeft": -6.403574, - "eMouseWheelNorth": -7.502186, - "eNoBlock": -6.809039, - "eSpeechTextOnly": -7.502186, - "eSpeechVoiceAndText": -6.809039, - "eSpeechVoiceOnly": -7.502186, - "else": -3.864600, - "enum": -6.809039, - "event": -6.809039, - "false": -4.557748, - "function": -3.813307, - "gControl_OnClick": -7.502186, - "gIconbar.Visible": -5.199601, - "gInventory.Visible": -7.502186, - "gPanel.Centre": -7.502186, - "gPanel.Visible": -5.556276, - "gRestartYN.Centre": -7.502186, - "gRestartYN.Visible": -6.809039, - "gRestoreGame.Visible": -6.809039, - "gSaveGame.Visible": -7.502186, - "game.debug_mode": -7.502186, - "gameSlotToSaveInto": -6.403574, - "i": -5.892749, - "if": -3.407842, - "import": -7.502186, - "initialize_control_panel": -7.502186, - "int": -4.668973, - "interface": -7.502186, - "interface_click": -7.502186, - "invCustomInv.ScrollDown": -7.502186, - "invCustomInv.ScrollUp": -7.502186, - "keycode": -4.668973, - "lblVoice.Visible": -7.502186, - "lstRestoreGamesList.FillSaveGameList": -7.502186, - "lstRestoreGamesList.SaveGameSlots": -7.502186, - "lstRestoreGamesList.SelectedIndex": -6.809039, - "lstSaveGamesList.FillSaveGameList": -7.502186, - "lstSaveGamesList.ItemCount": -6.809039, - "lstSaveGamesList.Items": -6.403574, - "lstSaveGamesList.SaveGameSlots": -6.809039, - "lstSaveGamesList.SelectedIndex": -6.403574, - "lstSaveGamesList_OnSelectionCh": -7.502186, - "mode": -6.403574, - "mouse.Mode": -6.115892, - "mouse.UseDefaultGraphic": -5.556276, - "mouse.UseModeGraphic": -5.892749, - "mouse.x": -7.502186, - "mouse.y": -7.502186, - "newdirection": -3.740986, - "null": -7.502186, - "on_event": -7.502186, - "on_key_press": -7.502186, - "on_mouse_click": -7.502186, - "player.ActiveInventory": -7.502186, - "player.PlaceOnWalkableArea": -7.502186, - "player.StopMoving": -6.403574, - "player.WalkStraight": -6.809039, - "player.on": -6.809039, - "player.x": -6.809039, - "player.y": -6.809039, - "repeatedly_execute": -7.502186, - "return": -6.809039, - "show_inventory_window": -7.502186, - "show_restore_game_dialog": -6.809039, - "show_save_game_dialog": -7.502186, - "sldAudio.Value": -7.502186, - "sldAudio_OnChange": -7.502186, - "sldGamma_OnChange": -7.502186, - "sldVoice.Visible": -7.502186, - "sldVoice_OnChange": -7.502186, - "static": -6.809039, - "struct": -7.502186, - "true": -5.104291, - "txtNewSaveName.Text": -5.892749, - "txtNewSaveName_OnActivate": -7.502186, - "while": -7.502186, - "{": -3.158381, - "||": -5.199601, - "}": -3.132739, + "!": -5.850405, + "#define": -7.236699, + "&&": -6.138087, + "(": -2.606836, + ")": -2.606836, + "*control": -4.710971, + "*theGui": -7.929846, + "+": -5.983936, + ",": -3.978603, + "-": -2.601970, + "//": -5.531951, + "//****************************************************************************************************": -5.850405, + "//But": -7.929846, + "//Ctrl": -7.929846, + "//START": -7.929846, + "//Use": -7.929846, + "//if": -7.929846, + ":": -7.236699, + ";": -2.776555, + "<": -7.929846, + "=": -1.114206, + ">": -7.236699, + "CONTROL": -7.929846, + "DISTANCE": -4.710971, + "Debug": -6.543552, + "DeleteSaveSlot": -7.929846, + "Display": -6.543552, + "ESC": -7.929846, + "EventType": -7.929846, + "FUNCTIONS": -7.929846, + "GUI": -7.929846, + "GUIControl": -4.710971, + "IsGamePaused": -6.831234, + "IsInterfaceEnabled": -7.236699, + "IsKeyPressed": -5.096633, + "IsSpeechVoxAvailable": -7.929846, + "KeyboardMovement": -7.236699, + "KeyboardMovement_CurrentDirection": -5.983936, + "KeyboardMovement_Directions": -6.543552, + "KeyboardMovement_KeyDown": -6.320409, + "KeyboardMovement_KeyDownLeft": -6.831234, + "KeyboardMovement_KeyDownRight": -6.831234, + "KeyboardMovement_KeyLeft": -6.320409, + "KeyboardMovement_KeyRight": -6.320409, + "KeyboardMovement_KeyStop": -6.831234, + "KeyboardMovement_KeyUp": -6.320409, + "KeyboardMovement_KeyUpLeft": -6.831234, + "KeyboardMovement_KeyUpRight": -6.831234, + "KeyboardMovement_Mode": -6.543552, + "KeyboardMovement_Modes": -6.543552, + "KeyboardMovement_VERSION": -7.929846, + "MouseButton": -4.838804, + "OF": -7.929846, + "PANEL": -7.929846, + "ProcessClick": -7.929846, + "QuitGame": -7.236699, + "RestoreGameSlot": -7.929846, + "SaveGameSlot": -7.929846, + "SetMode": -7.236699, + "SetVoiceMode": -6.543552, + "System.SupportsGammaControl": -7.929846, + "System.Volume": -7.929846, + "W": -7.929846, + "Wait": -6.831234, + "[": -6.138087, + "]": -6.138087, + "appreciate.": -7.929846, + "area": -7.929846, + "as": -7.929846, + "btnAbout_OnClick": -7.929846, + "btnCancelRestore_OnClick": -7.929846, + "btnCancelSave_OnClick": -7.929846, + "btnDeleteSave_OnClick": -7.929846, + "btnIconAbout_Click": -7.929846, + "btnIconCurInv_Click": -7.929846, + "btnIconExit_Click": -7.929846, + "btnIconInv_Click": -7.929846, + "btnIconLoad": -7.929846, + "btnIconLoad_Click": -7.236699, + "btnIconSave": -7.929846, + "btnIconSave_Click": -7.236699, + "btnInvDown_Click": -7.929846, + "btnInvOK_Click": -7.929846, + "btnInvSelect_Click": -7.929846, + "btnInvUp_Click": -7.929846, + "btnLoad_OnClick": -7.929846, + "btnQuit_OnClick": -7.929846, + "btnRestoreGame_OnClick": -7.929846, + "btnResume_OnClick": -7.929846, + "btnSaveGame_OnClick": -7.236699, + "btnSave_OnClick": -7.929846, + "btnVoice.Text": -5.983936, + "btnVoice.Visible": -7.929846, + "button": -4.751793, + "cEgo_Interact": -7.929846, + "cEgo_Look": -7.929846, + "cEgo_Talk": -7.929846, + "cancel": -7.929846, + "close_restore_game_dialog": -6.831234, + "close_save_game_dialog": -6.831234, + "control": -7.929846, + "data": -7.929846, + "dx": -4.934114, + "dy": -4.934114, + "eEventLeaveRoom": -7.929846, + "eKeyCtrlA": -7.929846, + "eKeyCtrlS": -7.929846, + "eKeyCtrlV": -7.929846, + "eKeyCtrlW": -7.929846, + "eKeyCtrlX": -7.929846, + "eKeyEscape": -7.929846, + "eKeyboardMovement_Down": -6.320409, + "eKeyboardMovement_DownLeft": -6.320409, + "eKeyboardMovement_DownRight": -6.320409, + "eKeyboardMovement_Left": -6.320409, + "eKeyboardMovement_None": -7.236699, + "eKeyboardMovement_Pressing": -7.236699, + "eKeyboardMovement_Right": -6.320409, + "eKeyboardMovement_Stop": -5.732622, + "eKeyboardMovement_Tapping": -7.236699, + "eKeyboardMovement_Up": -6.320409, + "eKeyboardMovement_UpLeft": -6.320409, + "eKeyboardMovement_UpRight": -6.320409, + "eModeInteract": -7.929846, + "eModePickup": -7.929846, + "eModePointer": -6.320409, + "eModeTalkto": -7.929846, + "eModeUseinv": -7.236699, + "eModeWalkto": -7.929846, + "eMouseLeft": -6.831234, + "eMouseWheelNorth": -7.929846, + "eNoBlock": -7.236699, + "eSpeechTextOnly": -7.929846, + "eSpeechVoiceAndText": -7.236699, + "eSpeechVoiceOnly": -7.929846, + "else": -4.292260, + "enum": -7.236699, + "event": -7.236699, + "false": -4.985407, + "function": -4.240967, + "gControl_OnClick": -7.929846, + "gIconbar.Visible": -5.627261, + "gInventory.Visible": -7.929846, + "gPanel.Centre": -7.929846, + "gPanel.Visible": -5.983936, + "gRestartYN.Centre": -7.929846, + "gRestartYN.Visible": -7.236699, + "gRestoreGame.Visible": -7.236699, + "gSaveGame.Visible": -7.929846, + "game.debug_mode": -7.929846, + "gameSlotToSaveInto": -6.831234, + "give": -7.929846, + "i": -6.320409, + "if": -3.835502, + "import": -7.929846, + "initialize_control_panel": -7.929846, + "int": -5.096633, + "interface": -7.929846, + "interface_click": -7.929846, + "invCustomInv.ScrollDown": -7.929846, + "invCustomInv.ScrollUp": -7.929846, + "is": -7.929846, + "keyboard": -7.929846, + "keycode": -4.985407, + "lblVoice.Visible": -7.929846, + "lstRestoreGamesList.FillSaveGameList": -7.929846, + "lstRestoreGamesList.SaveGameSlots": -7.929846, + "lstRestoreGamesList.SelectedIndex": -7.236699, + "lstSaveGamesList.FillSaveGameList": -7.929846, + "lstSaveGamesList.ItemCount": -7.236699, + "lstSaveGamesList.Items": -6.831234, + "lstSaveGamesList.SaveGameSlots": -7.236699, + "lstSaveGamesList.SelectedIndex": -6.831234, + "lstSaveGamesList_OnSelectionCh": -7.929846, + "mode": -6.831234, + "mouse.Mode": -6.320409, + "mouse.UseDefaultGraphic": -5.983936, + "mouse.UseModeGraphic": -6.320409, + "mouse.x": -7.929846, + "mouse.y": -7.929846, + "move": -7.929846, + "much": -7.929846, + "newdirection": -4.168646, + "null": -7.929846, + "on_event": -7.929846, + "on_key_press": -7.929846, + "on_mouse_click": -7.929846, + "player.ActiveInventory": -7.929846, + "player.PlaceOnWalkableArea": -7.929846, + "player.StopMoving": -6.831234, + "player.WalkStraight": -7.236699, + "player.on": -7.236699, + "player.x": -7.236699, + "player.y": -7.236699, + "players": -7.929846, + "repeatedly_execute": -7.929846, + "restart.": -7.929846, + "return": -7.236699, + "shortcuts": -7.929846, + "show_inventory_window": -7.929846, + "show_restore_game_dialog": -7.236699, + "show_save_game_dialog": -7.929846, + "sldAudio.Value": -7.929846, + "sldAudio_OnChange": -7.929846, + "sldGamma_OnChange": -7.929846, + "sldVoice.Visible": -7.929846, + "sldVoice_OnChange": -7.929846, + "some": -7.929846, + "standard": -7.929846, + "static": -7.236699, + "struct": -7.929846, + "this": -7.929846, + "to": -7.236699, + "true": -5.531951, + "txtNewSaveName.Text": -6.320409, + "txtNewSaveName_OnActivate": -7.929846, + "very": -7.929846, + "walkable": -7.929846, + "while": -7.929846, + "will": -7.236699, + "you": -7.929846, + "your": -7.929846, + "{": -3.586041, + "||": -5.531951, + "}": -3.560399, }, "AMPL": map[string]float64{ "(": -3.988984, @@ -1002,416 +1111,460 @@ var DefaultClassifier Classifier = &classifier{ "}": -2.932931, }, "API Blueprint": map[string]float64{ - "!": -4.960511, - "'": -6.059123, - "(": -2.840247, - ")": -3.420066, - "**": -6.059123, - "**API": -6.059123, - "**Note": -6.059123, - "**parse**": -6.059123, - "**real": -6.059123, - "+": -2.840247, - ",": -3.351073, - "-": -4.113213, - ":": -2.532763, - "A": -4.267364, - "API": -4.113213, - "APIs": -6.059123, - "Actions": -5.365976, - "Advanced": -5.365976, - "Also": -6.059123, - "As": -6.059123, - "Attributes": -4.960511, - "Blueprint": -4.672829, - "Blueprint**": -6.059123, - "Blueprint**.": -6.059123, - "Body": -6.059123, - "Date": -6.059123, - "FORMAT": -4.960511, - "Hello": -6.059123, - "ID.": -6.059123, - "In": -6.059123, - "JSON": -6.059123, - "Model": -6.059123, - "Next": -5.365976, - "One": -6.059123, - "Parameters": -4.672829, - "Previous": -5.365976, - "Raw": -4.672829, - "Resource": -4.960511, - "Response": -4.449685, - "Retrieves": -6.059123, - "The": -6.059123, - "This": -4.113213, - "Time": -6.059123, - "We": -6.059123, - "World": -6.059123, - "[": -3.574217, - "]": -3.574217, - "a": -4.113213, - "about": -6.059123, - "action": -5.365976, - "after": -6.059123, - "also": -6.059123, - "amount": -6.059123, - "an": -6.059123, - "and": -4.267364, - "another": -5.365976, - "application/json": -4.960511, - "apply": -6.059123, - "apply.": -6.059123, - "as": -6.059123, - "attributes": -6.059123, - "be": -6.059123, - "between": -6.059123, - "body": -5.365976, - "can": -5.365976, - "case": -6.059123, - "combined": -6.059123, - "complementary": -6.059123, - "contains": -6.059123, - "coupon": -4.672829, - "course": -6.059123, - "created": -6.059123, - "customer.": -6.059123, - "demonstrates": -5.365976, - "describe": -6.059123, - "description": -6.059123, - "discount": -5.365976, - "do": -6.059123, - "duplicate": -6.059123, - "every": -6.059123, - "example": -4.672829, - "examples": -6.059123, - "explain": -6.059123, - "fact": -6.059123, - "false": -5.365976, - "forget": -6.059123, - "given": -6.059123, - "going": -6.059123, - "how": -6.059123, - "https": -4.449685, - "id": -4.960511, - "in": -4.267364, - "information": -6.059123, - "installment": -6.059123, - "integer": -6.059123, - "is": -4.267364, - "it": -5.365976, - "keep": -6.059123, - "longer": -6.059123, - "message.": -6.059123, - "method": -6.059123, - "might": -6.059123, - "mind": -6.059123, - "next": -6.059123, - "no": -6.059123, - "not": -6.059123, - "null": -6.059123, - "number": -4.672829, - "object": -6.059123, - "of": -5.365976, - "off": -5.365976, - "on": -6.059123, - "one": -6.059123, - "or": -5.365976, - "parser": -6.059123, - "percent": -6.059123, - "percent_off": -6.059123, - "plain": -6.059123, - "please": -6.059123, - "positive": -6.059123, - "priority": -6.059123, - "progress": -6.059123, - "provided": -6.059123, - "redeem_by": -6.059123, - "redeemed": -6.059123, - "represents": -6.059123, - "request": -6.059123, - "resource": -4.960511, - "resource.": -6.059123, - "response": -6.059123, - "s": -6.059123, - "section.": -6.059123, - "simplest": -6.059123, - "single": -6.059123, - "stamp": -6.059123, - "state": -4.960511, - "status": -6.059123, - "string": -4.672829, - "such": -6.059123, - "text/plain": -6.059123, - "that": -4.960511, - "the": -3.420066, - "this": -5.365976, - "through": -6.059123, - "to": -4.113213, - "transition": -5.365976, - "transition.": -6.059123, - "true": -6.059123, - "view": -6.059123, - "want": -6.059123, - "we": -6.059123, - "what": -6.059123, - "which": -6.059123, - "will": -5.365976, - "with": -4.960511, - "written": -6.059123, - "you": -5.365976, - "{": -4.672829, - "}": -4.672829, - "–": -5.365976, + "!": -5.216746, + "#": -4.369448, + "##": -6.315358, + "%": -3.319626, + "'": -6.315358, + "(": -3.057261, + ")": -3.057261, + "**": -6.315358, + "**API": -6.315358, + "**Note": -6.315358, + "**parse**": -6.315358, + "**real": -6.315358, + "+": -3.096482, + ",": -3.542769, + "-": -3.917463, + ".": -5.622211, + "//github.com/apiaryio/drafter": -6.315358, + "//github.com/apiaryio/drafter#bindings": -6.315358, + "//raw.github.com/apiaryio/api": -4.929064, + ":": -2.760010, + "A": -4.523599, + "API": -4.118133, + "API.md": -5.622211, + "APIs": -6.315358, + "Action.md": -6.315358, + "Actions": -5.622211, + "Actions.md": -5.622211, + "Advanced": -4.705920, + "Also": -6.315358, + "As": -6.315358, + "Attributes": -5.216746, + "Attributes.md": -5.216746, + "Blueprint": -4.705920, + "Blueprint**": -6.315358, + "Blueprint**.": -6.315358, + "Body": -6.315358, + "Date": -6.315358, + "FORMAT": -5.216746, + "Github": -6.315358, + "Hello": -6.315358, + "ID.": -6.315358, + "In": -6.315358, + "JSON": -6.315358, + "Markdown": -6.315358, + "Model": -6.315358, + "Model.md": -6.315358, + "Next": -5.622211, + "One": -6.315358, + "Parameters": -4.929064, + "Parameters.md": -6.315358, + "Previous": -5.622211, + "Raw": -4.929064, + "Resource": -4.523599, + "Response": -4.705920, + "Retrieves": -6.315358, + "Simplest": -5.622211, + "The": -6.315358, + "This": -4.369448, + "Time": -6.315358, + "We": -6.315358, + "World": -6.315358, + "[": -3.750409, + "]": -3.750409, + "a": -4.369448, + "about": -6.315358, + "action": -5.622211, + "after": -6.315358, + "also": -6.315358, + "amount": -6.315358, + "an": -6.315358, + "and": -4.118133, + "another": -5.622211, + "application/json": -5.216746, + "apply": -6.315358, + "apply.": -6.315358, + "as": -5.622211, + "attributes": -5.622211, + "avoid": -6.315358, + "be": -6.315358, + "between": -6.315358, + "bindings": -6.315358, + "blueprint/master/examples/": -4.929064, + "body": -5.622211, + "can": -5.622211, + "case": -6.315358, + "code": -6.315358, + "combined": -6.315358, + "complementary": -6.315358, + "contains": -6.315358, + "coupon": -4.929064, + "course": -6.315358, + "created": -6.315358, + "customer.": -6.315358, + "demonstrate": -6.315358, + "demonstrates": -5.622211, + "describe": -6.315358, + "description": -6.315358, + "descriptions.": -6.315358, + "discount": -5.622211, + "do": -6.315358, + "duplicate": -6.315358, + "duplicates": -6.315358, + "every": -6.315358, + "example": -4.705920, + "examples": -6.315358, + "explain": -6.315358, + "fact": -6.315358, + "false": -5.622211, + "forget": -6.315358, + "given": -6.315358, + "going": -5.622211, + "how": -5.216746, + "https": -4.523599, + "id": -5.216746, + "in": -4.369448, + "information": -6.315358, + "installment": -6.315358, + "integer": -6.315358, + "is": -4.369448, + "it": -5.622211, + "its": -6.315358, + "just": -6.315358, + "keep": -6.315358, + "longer": -6.315358, + "message.": -6.315358, + "method": -6.315358, + "might": -6.315358, + "mind": -6.315358, + "next": -6.315358, + "no": -6.315358, + "not": -6.315358, + "null": -6.315358, + "number": -4.929064, + "object": -6.315358, + "of": -4.929064, + "off": -5.622211, + "on": -5.622211, + "one": -5.622211, + "opposed": -6.315358, + "or": -5.216746, + "output": -6.315358, + "parser": -6.315358, + "parser.": -6.315358, + "percent": -6.315358, + "percent_off": -6.315358, + "plain": -6.315358, + "please": -6.315358, + "positive": -6.315358, + "priority": -6.315358, + "progress": -6.315358, + "provided": -6.315358, + "really": -6.315358, + "redeem_by": -6.315358, + "redeemed": -6.315358, + "represents": -6.315358, + "request": -6.315358, + "resource": -5.216746, + "resource.": -6.315358, + "response": -6.315358, + "reuse": -6.315358, + "s": -6.315358, + "section.": -6.315358, + "see": -6.315358, + "seeing": -6.315358, + "simplest": -6.315358, + "single": -6.315358, + "stamp": -6.315358, + "state": -5.216746, + "status": -6.315358, + "string": -4.929064, + "such": -6.315358, + "text/plain": -6.315358, + "that": -5.216746, + "the": -3.482145, + "this": -5.622211, + "through": -6.315358, + "to": -3.917463, + "transition": -5.622211, + "transition.": -6.315358, + "true": -6.315358, + "view": -6.315358, + "want": -6.315358, + "we": -6.315358, + "what": -5.622211, + "which": -6.315358, + "will": -5.216746, + "with": -5.216746, + "written": -6.315358, + "you": -5.622211, + "{": -4.929064, + "}": -4.929064, + "–": -5.622211, }, "APL": map[string]float64{ - "(": -3.506736, - ")": -3.489037, - "+": -5.047181, - ",": -3.101271, - "-": -5.586178, - "/": -4.893031, - "/C": -7.532088, - "/Functions": -7.532088, - "/Page": -7.532088, - "/Z": -7.532088, - "/config": -7.532088, - "/input": -7.532088, - "/lines": -7.532088, - ":": -3.327396, - ";": -3.327396, - "": -7.532088, - "": -7.532088, - "": -7.532088, - "": -7.532088, - "": -6.838941, - "": -7.532088, - "": -7.532088, - "": -7.532088, - "
":           -7.532088,
-			"":           -7.546446,
+			"":           -6.853299,
+			"":              -7.546446,
+			"":             -7.546446,
+			"":             -7.546446,
+			"
":              -7.546446,
+			"":           -8.083020,
-			"":           -5.598113,
-			"":           -8.083020,
-			"":              -4.750815,
-			"":            -6.291260,
-			"":           -6.984408,
-			"":              -5.443963,
-			"":                -8.083020,
-			"

": -8.083020, - "": -6.003578, - "": -6.473582, - "
": -6.984408, - "