diff --git a/alias.go b/alias.go index d355612..ce4c2fc 100644 --- a/alias.go +++ b/alias.go @@ -2,11 +2,12 @@ package slinguist import "strings" -// GetLanguageByAlias returns the language related to the given alias or Otherlanguage otherwise. -func GetLanguageByAlias(alias string) (lang string) { +// GetLanguageByAlias returns the language related to the given alias and ok set to true, +// or Otherlanguage and ok set to false otherwise. +func GetLanguageByAlias(alias string) (lang string, ok bool) { a := strings.Split(alias, `,`)[0] a = strings.ToLower(a) - lang, ok := languagesByAlias[a] + lang, ok = languagesByAlias[a] if !ok { lang = OtherLanguage } diff --git a/alias_test.go b/alias_test.go index 9bdbb43..3612e37 100644 --- a/alias_test.go +++ b/alias_test.go @@ -6,21 +6,23 @@ func (s *TSuite) TestGetLanguageByAlias(c *C) { tests := []struct { alias string expectedLang string + expectedOk bool }{ - {alias: "BestLanguageEver", expectedLang: OtherLanguage}, - {alias: "aspx-vb", expectedLang: "ASP"}, - {alias: "C++", expectedLang: "C++"}, - {alias: "c++", expectedLang: "C++"}, - {alias: "objc", expectedLang: "Objective-C"}, - {alias: "golang", expectedLang: "Go"}, - {alias: "GOLANG", expectedLang: "Go"}, - {alias: "bsdmake", expectedLang: "Makefile"}, - {alias: "xhTmL", expectedLang: "HTML"}, - {alias: "python", expectedLang: "Python"}, + {alias: "BestLanguageEver", expectedLang: OtherLanguage, expectedOk: false}, + {alias: "aspx-vb", expectedLang: "ASP", expectedOk: true}, + {alias: "C++", expectedLang: "C++", expectedOk: true}, + {alias: "c++", expectedLang: "C++", expectedOk: true}, + {alias: "objc", expectedLang: "Objective-C", expectedOk: true}, + {alias: "golang", expectedLang: "Go", expectedOk: true}, + {alias: "GOLANG", expectedLang: "Go", expectedOk: true}, + {alias: "bsdmake", expectedLang: "Makefile", expectedOk: true}, + {alias: "xhTmL", expectedLang: "HTML", expectedOk: true}, + {alias: "python", expectedLang: "Python", expectedOk: true}, } for _, test := range tests { - lang := GetLanguageByAlias(test.alias) + lang, ok := GetLanguageByAlias(test.alias) c.Assert(lang, Equals, test.expectedLang) + c.Assert(ok, Equals, test.expectedOk) } } diff --git a/aliases_map.go b/aliases_map.go index bea090c..74d7f3a 100644 --- a/aliases_map.go +++ b/aliases_map.go @@ -2,7 +2,7 @@ package slinguist // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator // THIS FILE SHOULD NOT BE EDITED BY HAND -// Extracted from github/linguist commit: dae33dc2b20cddc85d1300435c3be7118a7115a9 +// Extracted from github/linguist commit: 60f864a138650dd17fafc94814be9ee2d3aaef8c // languagesByAlias keeps alias for different languages and use the name of the languages as a alias too. All the // keys (alias or not) are written in lower case and the whitespaces has been replaced by underscores. @@ -107,6 +107,7 @@ var languagesByAlias = map[string]string{ "clipper": "xBase", "clips": "CLIPS", "clojure": "Clojure", + "closure_templates": "Closure Templates", "cmake": "CMake", "cobol": "COBOL", "coffee": "CoffeeScript", @@ -137,6 +138,7 @@ var languagesByAlias = map[string]string{ "csv": "CSV", "cucumber": "Gherkin", "cuda": "Cuda", + "cweb": "CWeb", "cycript": "Cycript", "cython": "Cython", "d": "D", @@ -281,6 +283,7 @@ var languagesByAlias = map[string]string{ "jflex": "JFlex", "jison": "Jison", "jison_lex": "Jison Lex", + "jolie": "Jolie", "jruby": "Ruby", "js": "JavaScript", "json": "JSON", @@ -433,6 +436,7 @@ var languagesByAlias = map[string]string{ "pascal": "Pascal", "pasm": "Parrot Assembly", "pawn": "PAWN", + "pep8": "Pep8", "perl": "Perl", "perl6": "Perl6", "php": "PHP", @@ -529,6 +533,7 @@ var languagesByAlias = map[string]string{ "scss": "SCSS", "self": "Self", "sh": "Shell", + "shaderlab": "ShaderLab", "shell": "Shell", "shell-script": "Shell", "shellsession": "ShellSession", @@ -572,6 +577,7 @@ var languagesByAlias = map[string]string{ "textile": "Textile", "thrift": "Thrift", "ti_program": "TI Program", + "tl": "Type Language", "tla": "TLA", "toml": "TOML", "ts": "TypeScript", @@ -579,6 +585,7 @@ var languagesByAlias = map[string]string{ "turtle": "Turtle", "twig": "Twig", "txl": "TXL", + "type_language": "Type Language", "typescript": "TypeScript", "udiff": "Diff", "unified_parallel_c": "Unified Parallel C", diff --git a/classifier.go b/classifier.go new file mode 100644 index 0000000..db3117e --- /dev/null +++ b/classifier.go @@ -0,0 +1,100 @@ +package slinguist + +import ( + "math" + + "gopkg.in/src-d/simple-linguist.v1/internal/tokenizer" +) + +// GetLanguageByClassifier takes in a content and a list of candidates, and apply the classifier's Classify method to +// get the most probably language. If classifier is null then DefaultClassfier will be used. +func GetLanguageByClassifier(content []byte, candidates []string, classifier Classifier) string { + if classifier == nil { + classifier = DefaultClassifier + } + + scores := classifier.Classify(content, candidates) + if len(scores) == 0 { + return OtherLanguage + } + + return getLangugeHigherScore(scores) +} + +func getLangugeHigherScore(scores map[string]float64) string { + var language string + higher := -math.MaxFloat64 + for lang, score := range scores { + if higher < score { + language = lang + higher = score + } + } + + return language +} + +// Classifier is the interface that contains the method Classify which is in charge to assign scores to the possibles candidates. +// The scores must order the candidates so as the highest score be the most probably language of the content. +type Classifier interface { + Classify(content []byte, candidates []string) map[string]float64 +} + +type classifier struct { + languagesLogProbabilities map[string]float64 + tokensLogProbabilities map[string]map[string]float64 + tokensTotal float64 +} + +func (c *classifier) Classify(content []byte, candidates []string) map[string]float64 { + if len(content) == 0 { + return nil + } + + var languages []string + if len(candidates) == 0 { + languages = c.knownLangs() + } else { + languages = make([]string, 0, len(candidates)) + for _, candidate := range candidates { + if lang, ok := GetLanguageByAlias(candidate); ok { + languages = append(languages, lang) + } + } + } + + tokens := tokenizer.Tokenize(content) + scores := make(map[string]float64, len(languages)) + for _, language := range languages { + scores[language] = c.tokensLogProbability(tokens, language) + c.languagesLogProbabilities[language] + } + + return scores +} + +func (c *classifier) knownLangs() []string { + langs := make([]string, 0, len(c.languagesLogProbabilities)) + for lang := range c.languagesLogProbabilities { + langs = append(langs, lang) + } + + return langs +} + +func (c *classifier) tokensLogProbability(tokens []string, language string) float64 { + var sum float64 + for _, token := range tokens { + sum += c.tokenProbability(token, language) + } + + return sum +} + +func (c *classifier) tokenProbability(token, language string) float64 { + tokenProb, ok := c.tokensLogProbabilities[language][token] + if !ok { + tokenProb = math.Log(1.000000 / c.tokensTotal) + } + + return tokenProb +} diff --git a/classifier_test.go b/classifier_test.go new file mode 100644 index 0000000..f0787b3 --- /dev/null +++ b/classifier_test.go @@ -0,0 +1,32 @@ +package slinguist + +import ( + "io/ioutil" + "path/filepath" + + . "gopkg.in/check.v1" +) + +func (s *TSuite) TestGetLanguageByClassifier(c *C) { + const samples = `.linguist/samples/` + test := []struct { + filename string + candidates []string + expectedLang string + }{ + {filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"python", "ruby", "c", "c++"}, expectedLang: "C"}, + {filename: filepath.Join(samples, "C/blob.c"), candidates: nil, expectedLang: "C"}, + {filename: filepath.Join(samples, "C/main.c"), candidates: nil, expectedLang: "C"}, + {filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"python", "ruby", "c++"}, expectedLang: "C++"}, + {filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"ruby"}, expectedLang: "Ruby"}, + {filename: filepath.Join(samples, "Python/django-models-base.py"), candidates: []string{"python", "ruby", "c", "c++"}, expectedLang: "Python"}, + {filename: filepath.Join(samples, "Python/django-models-base.py"), candidates: nil, expectedLang: "Python"}, + } + + for _, test := range test { + content, err := ioutil.ReadFile(test.filename) + c.Assert(err, Equals, nil) + lang := GetLanguageByClassifier(content, test.candidates, nil) + c.Assert(lang, Equals, test.expectedLang) + } +} diff --git a/common.go b/common.go index 5935ad3..2262e7d 100644 --- a/common.go +++ b/common.go @@ -52,7 +52,11 @@ func GetLanguage(filename string, content []byte) string { return lang } - lang, _ := GetLanguageByContent(filename, content) + if lang, safe := GetLanguageByContent(filename, content); safe { + return lang + } + + lang := GetLanguageByClassifier(content, nil, nil) return lang } diff --git a/content.go b/content.go index 2cef019..36f8973 100644 --- a/content.go +++ b/content.go @@ -2,7 +2,7 @@ package slinguist // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator // THIS FILE SHOULD NOT BE EDITED BY HAND -// Extracted from github/linguist commit: dae33dc2b20cddc85d1300435c3be7118a7115a9 +// Extracted from github/linguist commit: 60f864a138650dd17fafc94814be9ee2d3aaef8c import ( "path/filepath" @@ -117,8 +117,8 @@ var matchers = map[string]languageMatcher{ return "Forth", true } else if f_FilebenchWML_Matcher_0.Match(i) { return "Filebench WML", true - } else if f_FORTRAN_Matcher_0.Match(i) { - return "FORTRAN", true + } else if f_Fortran_Matcher_0.Match(i) { + return "Fortran", true } return OtherLanguage, false @@ -126,8 +126,8 @@ var matchers = map[string]languageMatcher{ ".for": func(i []byte) (string, bool) { if for_Forth_Matcher_0.Match(i) { return "Forth", true - } else if for_FORTRAN_Matcher_0.Match(i) { - return "FORTRAN", true + } else if for_Fortran_Matcher_0.Match(i) { + return "Fortran", true } return OtherLanguage, false @@ -184,8 +184,8 @@ var matchers = map[string]languageMatcher{ return "Common Lisp", true } else if l_Lex_Matcher_0.Match(i) { return "Lex", true - } else if l_Groff_Matcher_0.Match(i) { - return "Groff", true + } else if l_Roff_Matcher_0.Match(i) { + return "Roff", true } else if l_PicoLisp_Matcher_0.Match(i) { return "PicoLisp", true } @@ -239,8 +239,8 @@ var matchers = map[string]languageMatcher{ ".md": func(i []byte) (string, bool) { if md_Markdown_Matcher_0.Match(i) || md_Markdown_Matcher_1.Match(i) { return "Markdown", true - } else if md_GCCmachinedescription_Matcher_0.Match(i) { - return "GCC machine description", true + } else if md_GCCMachineDescription_Matcher_0.Match(i) { + return "GCC Machine Description", true } return "Markdown", true @@ -264,15 +264,15 @@ var matchers = map[string]languageMatcher{ return "Linux Kernel Module", false }, ".ms": func(i []byte) (string, bool) { - if ms_Groff_Matcher_0.Match(i) { - return "Groff", true + if ms_Roff_Matcher_0.Match(i) { + return "Roff", true } return "MAXScript", true }, ".n": func(i []byte) (string, bool) { - if n_Groff_Matcher_0.Match(i) { - return "Groff", true + if n_Roff_Matcher_0.Match(i) { + return "Roff", true } else if n_Nemerle_Matcher_0.Match(i) { return "Nemerle", true } @@ -314,19 +314,10 @@ var matchers = map[string]languageMatcher{ return OtherLanguage, false }, ".pm": func(i []byte) (string, bool) { - if pm_Perl_Matcher_0.Match(i) { - return "Perl", true - } else if pm_Perl6_Matcher_0.Match(i) { + if pm_Perl6_Matcher_0.Match(i) { return "Perl6", true - } - - return OtherLanguage, false - }, - ".t": func(i []byte) (string, bool) { - if t_Perl_Matcher_0.Match(i) { + } else if pm_Perl_Matcher_0.Match(i) { return "Perl", true - } else if t_Perl6_Matcher_0.Match(i) { - return "Perl6", true } return OtherLanguage, false @@ -372,8 +363,8 @@ var matchers = map[string]languageMatcher{ ".rno": func(i []byte) (string, bool) { if rno_RUNOFF_Matcher_0.Match(i) { return "RUNOFF", true - } else if rno_Groff_Matcher_0.Match(i) { - return "Groff", true + } else if rno_Roff_Matcher_0.Match(i) { + return "Roff", true } return OtherLanguage, false @@ -423,6 +414,17 @@ var matchers = map[string]languageMatcher{ return OtherLanguage, false }, + ".t": func(i []byte) (string, bool) { + if t_Turing_Matcher_0.Match(i) { + return "Turing", true + } else if t_Perl6_Matcher_0.Match(i) { + return "Perl6", true + } else if t_Perl_Matcher_0.Match(i) { + return "Perl", true + } + + return OtherLanguage, false + }, ".toc": func(i []byte) (string, bool) { if toc_WorldofWarcraftAddonData_Matcher_0.Match(i) { return "World of Warcraft Addon Data", true @@ -481,9 +483,9 @@ var ( es_Erlang_Matcher_0 = regexp.MustCompile(`(?m)^\s*(?:%%|main\s*\(.*?\)\s*->)`) f_Forth_Matcher_0 = regexp.MustCompile(`(?m)^: `) f_FilebenchWML_Matcher_0 = regexp.MustCompile(`(?m)flowop`) - f_FORTRAN_Matcher_0 = regexp.MustCompile(`(?mi)^([c*][^abd-z]| (subroutine|program|end|data)\s|\s*!)`) + f_Fortran_Matcher_0 = regexp.MustCompile(`(?mi)^([c*][^abd-z]| (subroutine|program|end|data)\s|\s*!)`) for_Forth_Matcher_0 = regexp.MustCompile(`(?m)^: `) - for_FORTRAN_Matcher_0 = regexp.MustCompile(`(?mi)^([c*][^abd-z]| (subroutine|program|end|data)\s|\s*!)`) + for_Fortran_Matcher_0 = regexp.MustCompile(`(?mi)^([c*][^abd-z]| (subroutine|program|end|data)\s|\s*!)`) fr_Forth_Matcher_0 = regexp.MustCompile(`(?m)^(: |also |new-device|previous )`) fr_Frege_Matcher_0 = regexp.MustCompile(`(?m)^\s*(import|module|package|data|type) `) fs_Forth_Matcher_0 = regexp.MustCompile(`(?m)^(: |new-device)`) @@ -503,7 +505,7 @@ var ( inc_POVDashRaySDL_Matcher_0 = regexp.MustCompile(`(?m)^\s*#(declare|local|macro|while)\s`) l_CommonLisp_Matcher_0 = regexp.MustCompile(`(?m)\(def(un|macro)\s`) l_Lex_Matcher_0 = regexp.MustCompile(`(?m)^(%[%{}]xs|<.*>)`) - l_Groff_Matcher_0 = regexp.MustCompile(`(?mi)^\.[a-z][a-z](\s|$)`) + l_Roff_Matcher_0 = regexp.MustCompile(`(?mi)^\.[a-z][a-z](\s|$)`) l_PicoLisp_Matcher_0 = regexp.MustCompile(`(?m)^\((de|class|rel|code|data|must)\s`) ls_LoomScript_Matcher_0 = regexp.MustCompile(`(?m)^\s*package\s*[\w\.\/\*\s]*\s*{`) lsp_CommonLisp_Matcher_0 = regexp.MustCompile(`(?mi)^\s*\((defun|in-package|defpackage) `) @@ -519,14 +521,14 @@ var ( m_Limbo_Matcher_0 = regexp.MustCompile(`(?m)^\w+\s*:\s*module\s*{`) md_Markdown_Matcher_0 = regexp.MustCompile(`(?mi)(^[-a-z0-9=#!\*\[|>])|<\/`) md_Markdown_Matcher_1 = regexp.MustCompile(`(?m)^$`) - md_GCCmachinedescription_Matcher_0 = regexp.MustCompile(`(?m)^(;;|\(define_)`) + md_GCCMachineDescription_Matcher_0 = regexp.MustCompile(`(?m)^(;;|\(define_)`) ml_OCaml_Matcher_0 = regexp.MustCompile(`(?m)(^\s*module)|let rec |match\s+(\S+\s)+with`) ml_StandardML_Matcher_0 = regexp.MustCompile(`(?m)=> |case\s+(\S+\s)+of`) mod_XML_Matcher_0 = regexp.MustCompile(`(?m))\s*(\d{2}:\d{2}:\d{2},\d{3})$`) + t_Turing_Matcher_0 = regexp.MustCompile(`(?m)^\s*%[ \t]+|^\s*var\s+\w+\s*:=\s*\w+`) + t_Perl6_Matcher_0 = regexp.MustCompile(`(?m)^\s*(?:use\s+v6\s*;|\bmodule\b|\b(?:my\s+)?class\b)`) + t_Perl_Matcher_0 = regexp.MustCompile(`(?m)\buse\s+(?:strict\b|v?5\.)`) toc_WorldofWarcraftAddonData_Matcher_0 = regexp.MustCompile(`(?m)^## |@no-lib-strip@`) toc_TeX_Matcher_0 = regexp.MustCompile(`(?m)^\\(contentsline|defcounter|beamer|boolfalse)`) ts_XML_Matcher_0 = regexp.MustCompile(`(?m)": -6.723516, + "?": -8.974808, + "DOM": -8.974808, + "DOM.": -9.667955, + "HTML": -8.974808, + "HTML.": -8.974808, + "Null": -9.667955, + "XML": -7.270060, + "XML.": -7.722045, + "_": -7.470730, + "А": -8.281661, + "Б": -8.058517, + "В": -4.698142, + "Г": -8.281661, + "Д": -4.698142, + "Е": -5.114078, + "Ж": -9.667955, + "З": -4.586551, + "И": -5.311246, + "К": -4.311369, + "Л": -6.233968, + "М": -6.266758, + "Н": -4.586551, + "О": -4.618099, + "П": -3.984375, + "Р": -5.839314, + "С": -4.431513, + "Т": -4.400097, + "У": -6.004393, + "Ф": -5.979076, + "Х": -8.974808, + "Ц": -5.607512, + "Ч": -6.576913, + "Ш": -6.623433, + "Э": -5.796754, + "а": -2.500146, + "б": -4.274328, + "в": -3.881058, + "г": -4.808143, + "д": -4.036743, + "е": -2.538658, + "ж": -5.146166, + "з": -4.839641, + "и": -2.864450, + "й": -4.684348, + "к": -3.479691, + "л": -3.220649, + "м": -4.076968, + "н": -2.950150, + "о": -2.582054, + "п": -4.301979, + "р": -3.197156, + "с": -3.578910, + "т": -2.868899, + "у": -4.029600, + "ф": -5.817807, + "х": -6.202219, + "ц": -4.306663, + "ч": -4.657320, + "ш": -7.470730, + "щ": -5.524820, + "ъ": -5.979076, + "ы": -3.793024, + "ь": -4.043938, + "ю": -6.112607, + "я": -4.562010, + "\ufeff": -8.974808, + }, + "ABAP": map[string]float64{ + "!": -3.263996, + "(": -4.442651, + ")": -4.442651, + "*": -2.427748, + "+": -4.219508, + "-": -4.219508, + ".": -3.631721, + "<-()]>": -5.135798, + "": -5.135798, + "": -4.442651, + "": -4.730333, + "=": -3.526361, + ">": -4.442651, + "CLASS": -5.135798, + "CL_CSV_PARSER": -4.219508, + "CONSTRUCTOR": -5.828946, + "CSV": -5.828946, + "CX": -5.828946, + "CX_CSV_PARSE_ERROR": -5.828946, + "DEFINITION": -5.135798, + "Get": -5.828946, + "IMPLEMENTATION": -5.828946, + "Instance": -5.135798, + "Method": -5.135798, + "Parse": -5.828946, + "Private": -5.828946, + "Public": -5.828946, + "RETURNING": -5.828946, + "SIGNATURE": -5.135798, + "STRINGTAB": -4.730333, + "Space": -5.135798, + "TYPE": -4.730333, + "This": -5.828946, + "[": -4.219508, + "_LINES": -5.828946, + "_csvstring": -5.135798, + "_delegate": -5.828946, + "_lines": -5.828946, + "_lines.": -5.828946, + "_parse_line": -5.135798, + "_separator": -5.828946, + "_skip_first_line": -5.828946, + "_textindicator": -5.828946, + "`": -4.442651, + "abap": -5.828946, + "abap_bool": -5.135798, + "abap_true.": -5.135798, + "an": -5.828946, + "append": -5.135798, + "assigning": -5.828946, + "at": -5.135798, + "c": -5.135798, + "char": -5.135798, + "cl_abap_char_utilities": -5.828946, + "cl_csv_parser": -5.135798, + "cl_object": -5.828946, + "class": -5.135798, + "clear": -5.828946, + "concatenate": -4.442651, + "constants": -5.828946, + "constructor": -5.135798, + "constructor.": -5.828946, + "cr_lf": -5.828946, + "create": -5.828946, + "csv": -5.828946, + "csvstring": -5.828946, + "csvstring.": -5.828946, + "csvvalue": -4.037186, + "csvvalue.": -4.219508, + "csvvalues.": -5.135798, + "cx_csv_parse_error": -5.135798, + "data": -4.730333, + "definition": -5.828946, + "delegate": -5.828946, + "delegate.": -5.828946, + "do": -4.730333, + "e": -5.828946, + "else.": -4.442651, + "endclass.": -5.828946, + "endif.": -4.037186, + "endmethod.": -5.135798, + "endwhile.": -5.135798, + "error": -5.828946, + "exception": -5.828946, + "exporting": -5.828946, + "field": -5.828946, + "files": -4.730333, + "final": -5.828946, + "formatting": -5.828946, + "from": -5.828946, + "here": -4.730333, + "if_csv_parser_delegate": -5.828946, + "implementation.": -5.828946, + "importing": -5.828946, + "in": -5.828946, + "include": -4.730333, + "indicates": -5.828946, + "inheriting": -5.828946, + "into": -4.037186, + "is_first_line": -5.828946, + "line": -5.828946, + "lines": -4.442651, + "loop": -5.828946, + "message": -5.135798, + "method": -5.135798, + "methods": -5.135798, + "msg.": -5.135798, + "not": -4.730333, + "of": -5.135798, + "other": -4.730333, + "parse": -5.135798, + "pools": -5.828946, + "pos": -5.135798, + "private": -5.828946, + "protected": -5.828946, + "public": -4.730333, + "raise": -5.828946, + "raising": -5.828946, + "ref": -5.828946, + "returning.": -5.828946, + "section.": -4.730333, + "separator": -5.828946, + "separator.": -5.828946, + "skip_first_line": -5.828946, + "skip_first_line.": -5.828946, + "source": -4.730333, + "split": -5.828946, + "standard": -5.135798, + "string": -5.828946, + "string.": -4.730333, + "super": -5.828946, + "symbols": -5.828946, + "table": -4.730333, + "text_ended": -5.828946, + "the": -4.730333, + "to": -4.730333, + "type": -3.431050, + "value": -5.135798, + "values": -5.135798, + "|": -3.883035, + }, + "ABNF": map[string]float64{ + "(": -3.938340, + ")": -4.056123, + "*": -4.631487, + "*DIGIT": -6.828712, + "*basic": -6.135565, + "*literal": -6.828712, + "*newline": -6.828712, + "*non": -6.828712, + ",": -5.036953, + "-": -1.576439, + "/": -3.165150, + ":": -5.442418, + ";": -2.821379, + "=": -2.434263, + "ABNF": -6.135565, + "ALPHA": -6.135565, + "Array": -6.135565, + "Basic": -5.730100, + "Boolean": -6.828712, + "Built": -6.828712, + "Comment": -6.828712, + "DIGIT": -4.120662, + "Datetime": -6.828712, + "Float": -6.828712, + "HEXDIG": -6.828712, + "Inline": -6.828712, + "Integer": -6.828712, + "Key": -6.828712, + "License": -6.828712, + "Literal": -6.135565, + "MIT": -6.828712, + "Multiline": -6.135565, + "Newline": -6.828712, + "RFC": -6.135565, + "See": -6.828712, + "Source": -6.828712, + "Standard": -6.828712, + "String": -5.219274, + "Strings": -6.828712, + "TOML": -6.135565, + "Table": -5.442418, + "This": -6.828712, + "Value": -6.828712, + "Whitespace": -6.828712, + "[": -4.631487, + "]": -4.631487, + "according": -6.828712, + "an": -6.828712, + "apostraphe": -5.036953, + "array": -3.938340, + "as": -6.828712, + "attempt": -6.828712, + "based": -6.135565, + "basic": -4.189655, + "body": -5.442418, + "boolean": -6.135565, + "char": -4.749271, + "clarity": -6.828712, + "close": -4.749271, + "comment": -4.749271, + "date": -4.526127, + "decimal": -6.135565, + "define": -6.828712, + "defined": -6.135565, + "delim": -5.036953, + "digit": -6.135565, + "e": -6.135565, + "empty": -5.730100, + "eol": -6.135565, + "escape": -6.135565, + "escaped": -6.828712, + "exp": -5.730100, + "expression": -5.730100, + "false": -6.135565, + "float": -6.135565, + "for": -6.828712, + "frac": -5.730100, + "full": -5.442418, + "fullyear": -6.135565, + "grammar": -6.828712, + "here": -6.828712, + "hour": -5.730100, + "http": -6.828712, + "https": -6.828712, + "in": -5.442418, + "inline": -4.263763, + "int": -5.442418, + "integer": -5.442418, + "is": -6.828712, + "key": -4.120662, + "keyval": -5.036953, + "keyvals": -5.219274, + "leap": -6.828712, + "literal": -4.343805, + "mark": -4.749271, + "mday": -6.135565, + "minus": -6.135565, + "minute": -5.730100, + "ml": -3.832980, + "month": -6.135565, + "month/year": -6.828712, + "newline": -5.219274, + "newlines": -5.219274, + "non": -5.442418, + "numoffset": -6.135565, + "offset": -6.135565, + "on": -6.135565, + "open": -4.749271, + "pairs": -6.828712, + "partial": -6.135565, + "plus": -6.135565, + "point": -6.135565, + "prefixable": -6.135565, + "quotation": -4.749271, + "quoted": -6.135565, + "reproduced": -6.828712, + "rules": -6.828712, + "secfrac": -6.135565, + "second": -5.730100, + "sep": -4.343805, + "start": -6.135565, + "std": -5.036953, + "string": -4.056123, + "symbol": -6.135565, + "table": -3.427515, + "terms": -6.828712, + "the": -6.828712, + "time": -3.832980, + "to": -6.135565, + "toml": -6.828712, + "true": -6.135565, + "underscore": -5.730100, + "unescaped": -5.730100, + "unquoted": -6.135565, + "val": -5.036953, + "values": -5.730100, + "ws": -4.120662, + "zero": -6.135565, + }, + "AGS Script": map[string]float64{ + "!": -5.422745, + "&&": -5.710427, + "(": -2.184066, + ")": -2.184066, + "*control": -4.283311, + "*theGui": -7.502186, + "+": -5.556276, + ",": -3.610366, + "-": -5.017280, + ":": -6.809039, + ";": -2.354692, + "<": -7.502186, + "=": -2.169468, + ">": -6.809039, + "DISTANCE": -4.324133, + "Debug": -6.115892, + "DeleteSaveSlot": -7.502186, + "Display": -6.115892, + "EventType": -7.502186, + "GUI": -7.502186, + "GUIControl": -4.283311, + "IsGamePaused": -6.403574, + "IsInterfaceEnabled": -6.809039, + "IsKeyPressed": -4.668973, + "IsSpeechVoxAvailable": -7.502186, + "KeyboardMovement": -6.809039, + "KeyboardMovement_CurrentDirection": -5.556276, + "KeyboardMovement_Directions": -6.115892, + "KeyboardMovement_KeyDown": -5.892749, + "KeyboardMovement_KeyDownLeft": -6.403574, + "KeyboardMovement_KeyDownRight": -6.403574, + "KeyboardMovement_KeyLeft": -5.892749, + "KeyboardMovement_KeyRight": -5.892749, + "KeyboardMovement_KeyStop": -6.403574, + "KeyboardMovement_KeyUp": -5.892749, + "KeyboardMovement_KeyUpLeft": -6.403574, + "KeyboardMovement_KeyUpRight": -6.403574, + "KeyboardMovement_Mode": -6.115892, + "KeyboardMovement_Modes": -6.115892, + "MouseButton": -4.411144, + "ProcessClick": -7.502186, + "QuitGame": -6.809039, + "RestoreGameSlot": -7.502186, + "SaveGameSlot": -7.502186, + "SetMode": -6.809039, + "SetVoiceMode": -6.115892, + "System.SupportsGammaControl": -7.502186, + "System.Volume": -7.502186, + "Wait": -6.403574, + "[": -5.710427, + "]": -5.710427, + "btnAbout_OnClick": -7.502186, + "btnCancelRestore_OnClick": -7.502186, + "btnCancelSave_OnClick": -7.502186, + "btnDeleteSave_OnClick": -7.502186, + "btnIconAbout_Click": -7.502186, + "btnIconCurInv_Click": -7.502186, + "btnIconExit_Click": -7.502186, + "btnIconInv_Click": -7.502186, + "btnIconLoad": -7.502186, + "btnIconLoad_Click": -6.809039, + "btnIconSave": -7.502186, + "btnIconSave_Click": -6.809039, + "btnInvDown_Click": -7.502186, + "btnInvOK_Click": -7.502186, + "btnInvSelect_Click": -7.502186, + "btnInvUp_Click": -7.502186, + "btnLoad_OnClick": -7.502186, + "btnQuit_OnClick": -7.502186, + "btnRestoreGame_OnClick": -7.502186, + "btnResume_OnClick": -7.502186, + "btnSaveGame_OnClick": -6.809039, + "btnSave_OnClick": -7.502186, + "btnVoice.Text": -5.556276, + "btnVoice.Visible": -7.502186, + "button": -4.324133, + "cEgo_Interact": -7.502186, + "cEgo_Look": -7.502186, + "cEgo_Talk": -7.502186, + "close_restore_game_dialog": -6.403574, + "close_save_game_dialog": -6.403574, + "control": -7.502186, + "data": -7.502186, + "dx": -4.506454, + "dy": -4.506454, + "eEventLeaveRoom": -7.502186, + "eKeyCtrlA": -7.502186, + "eKeyCtrlS": -7.502186, + "eKeyCtrlV": -7.502186, + "eKeyCtrlW": -7.502186, + "eKeyCtrlX": -7.502186, + "eKeyEscape": -7.502186, + "eKeyboardMovement_Down": -5.892749, + "eKeyboardMovement_DownLeft": -5.892749, + "eKeyboardMovement_DownRight": -5.892749, + "eKeyboardMovement_Left": -5.892749, + "eKeyboardMovement_None": -6.809039, + "eKeyboardMovement_Pressing": -6.809039, + "eKeyboardMovement_Right": -5.892749, + "eKeyboardMovement_Stop": -5.304962, + "eKeyboardMovement_Tapping": -6.809039, + "eKeyboardMovement_Up": -5.892749, + "eKeyboardMovement_UpLeft": -5.892749, + "eKeyboardMovement_UpRight": -5.892749, + "eModeInteract": -7.502186, + "eModePointer": -5.892749, + "eModeTalkto": -7.502186, + "eModeUseinv": -6.809039, + "eModeWalkto": -7.502186, + "eMouseLeft": -6.403574, + "eMouseWheelNorth": -7.502186, + "eNoBlock": -6.809039, + "eSpeechTextOnly": -7.502186, + "eSpeechVoiceAndText": -6.809039, + "eSpeechVoiceOnly": -7.502186, + "else": -3.864600, + "enum": -6.809039, + "event": -6.809039, + "false": -4.557748, + "function": -3.813307, + "gControl_OnClick": -7.502186, + "gIconbar.Visible": -5.199601, + "gInventory.Visible": -7.502186, + "gPanel.Centre": -7.502186, + "gPanel.Visible": -5.556276, + "gRestartYN.Centre": -7.502186, + "gRestartYN.Visible": -6.809039, + "gRestoreGame.Visible": -6.809039, + "gSaveGame.Visible": -7.502186, + "game.debug_mode": -7.502186, + "gameSlotToSaveInto": -6.403574, + "i": -5.892749, + "if": -3.407842, + "import": -7.502186, + "initialize_control_panel": -7.502186, + "int": -4.668973, + "interface": -7.502186, + "interface_click": -7.502186, + "invCustomInv.ScrollDown": -7.502186, + "invCustomInv.ScrollUp": -7.502186, + "keycode": -4.668973, + "lblVoice.Visible": -7.502186, + "lstRestoreGamesList.FillSaveGameList": -7.502186, + "lstRestoreGamesList.SaveGameSlots": -7.502186, + "lstRestoreGamesList.SelectedIndex": -6.809039, + "lstSaveGamesList.FillSaveGameList": -7.502186, + "lstSaveGamesList.ItemCount": -6.809039, + "lstSaveGamesList.Items": -6.403574, + "lstSaveGamesList.SaveGameSlots": -6.809039, + "lstSaveGamesList.SelectedIndex": -6.403574, + "lstSaveGamesList_OnSelectionCh": -7.502186, + "mode": -6.403574, + "mouse.Mode": -6.115892, + "mouse.UseDefaultGraphic": -5.556276, + "mouse.UseModeGraphic": -5.892749, + "mouse.x": -7.502186, + "mouse.y": -7.502186, + "newdirection": -3.740986, + "null": -7.502186, + "on_event": -7.502186, + "on_key_press": -7.502186, + "on_mouse_click": -7.502186, + "player.ActiveInventory": -7.502186, + "player.PlaceOnWalkableArea": -7.502186, + "player.StopMoving": -6.403574, + "player.WalkStraight": -6.809039, + "player.on": -6.809039, + "player.x": -6.809039, + "player.y": -6.809039, + "repeatedly_execute": -7.502186, + "return": -6.809039, + "show_inventory_window": -7.502186, + "show_restore_game_dialog": -6.809039, + "show_save_game_dialog": -7.502186, + "sldAudio.Value": -7.502186, + "sldAudio_OnChange": -7.502186, + "sldGamma_OnChange": -7.502186, + "sldVoice.Visible": -7.502186, + "sldVoice_OnChange": -7.502186, + "static": -6.809039, + "struct": -7.502186, + "true": -5.104291, + "txtNewSaveName.Text": -5.892749, + "txtNewSaveName_OnActivate": -7.502186, + "while": -7.502186, + "{": -3.158381, + "||": -5.199601, + "}": -3.132739, + }, + "AMPL": map[string]float64{ + "(": -3.988984, + ")": -3.988984, + "*": -4.276666, + "+": -6.068426, + ",": -2.667228, + "-": -6.068426, + "..": -4.969813, + ":": -3.123987, + ";": -2.602690, + "<": -6.068426, + "=": -3.072693, + ">": -3.765840, + "BEAMS": -4.122515, + "COLUMNS": -4.276666, + "I": -4.122515, + "KnapsackBound": -4.969813, + "ROWS": -4.276666, + "S": -4.969813, + "T": -4.969813, + "Take": -4.969813, + "TotalValue": -6.068426, + "Value": -4.969813, + "Weight": -4.969813, + "WeightLimit": -6.068426, + "X": -4.458988, + "[": -3.178054, + "]": -3.178054, + "beam_values": -4.458988, + "binary": -6.068426, + "critical_area": -4.458988, + "critical_limit": -6.068426, + "critical_max": -5.375278, + "critical_values": -5.375278, + "data": -6.068426, + "h": -3.072693, + "i": -3.123987, + "in": -3.123987, + "integer": -5.375278, + "k": -3.072693, + "maximize": -5.375278, + "minimize": -4.969813, + "num_beams": -5.375278, + "num_cols": -5.375278, + "num_rows": -5.375278, + "param": -3.503476, + "s.t.": -6.068426, + "set": -4.276666, + "subject": -5.375278, + "sum": -3.765840, + "to": -5.375278, + "total_critical_dosage": -6.068426, + "total_critical_slack": -6.068426, + "total_tumor_dosage": -6.068426, + "total_tumor_slack": -6.068426, + "tumor_area": -4.458988, + "tumor_limit": -6.068426, + "tumor_min": -5.375278, + "tumor_values": -5.375278, + "var": -4.682131, + "{": -2.932931, + "}": -2.932931, + }, + "API Blueprint": map[string]float64{ + "!": -4.960511, + "'": -6.059123, + "(": -2.840247, + ")": -3.420066, + "**": -6.059123, + "**API": -6.059123, + "**Note": -6.059123, + "**parse**": -6.059123, + "**real": -6.059123, + "+": -2.840247, + ",": -3.351073, + "-": -4.113213, + ":": -2.532763, + "A": -4.267364, + "API": -4.113213, + "APIs": -6.059123, + "Actions": -5.365976, + "Advanced": -5.365976, + "Also": -6.059123, + "As": -6.059123, + "Attributes": -4.960511, + "Blueprint": -4.672829, + "Blueprint**": -6.059123, + "Blueprint**.": -6.059123, + "Body": -6.059123, + "Date": -6.059123, + "FORMAT": -4.960511, + "Hello": -6.059123, + "ID.": -6.059123, + "In": -6.059123, + "JSON": -6.059123, + "Model": -6.059123, + "Next": -5.365976, + "One": -6.059123, + "Parameters": -4.672829, + "Previous": -5.365976, + "Raw": -4.672829, + "Resource": -4.960511, + "Response": -4.449685, + "Retrieves": -6.059123, + "The": -6.059123, + "This": -4.113213, + "Time": -6.059123, + "We": -6.059123, + "World": -6.059123, + "[": -3.574217, + "]": -3.574217, + "a": -4.113213, + "about": -6.059123, + "action": -5.365976, + "after": -6.059123, + "also": -6.059123, + "amount": -6.059123, + "an": -6.059123, + "and": -4.267364, + "another": -5.365976, + "application/json": -4.960511, + "apply": -6.059123, + "apply.": -6.059123, + "as": -6.059123, + "attributes": -6.059123, + "be": -6.059123, + "between": -6.059123, + "body": -5.365976, + "can": -5.365976, + "case": -6.059123, + "combined": -6.059123, + "complementary": -6.059123, + "contains": -6.059123, + "coupon": -4.672829, + "course": -6.059123, + "created": -6.059123, + "customer.": -6.059123, + "demonstrates": -5.365976, + "describe": -6.059123, + "description": -6.059123, + "discount": -5.365976, + "do": -6.059123, + "duplicate": -6.059123, + "every": -6.059123, + "example": -4.672829, + "examples": -6.059123, + "explain": -6.059123, + "fact": -6.059123, + "false": -5.365976, + "forget": -6.059123, + "given": -6.059123, + "going": -6.059123, + "how": -6.059123, + "https": -4.449685, + "id": -4.960511, + "in": -4.267364, + "information": -6.059123, + "installment": -6.059123, + "integer": -6.059123, + "is": -4.267364, + "it": -5.365976, + "keep": -6.059123, + "longer": -6.059123, + "message.": -6.059123, + "method": -6.059123, + "might": -6.059123, + "mind": -6.059123, + "next": -6.059123, + "no": -6.059123, + "not": -6.059123, + "null": -6.059123, + "number": -4.672829, + "object": -6.059123, + "of": -5.365976, + "off": -5.365976, + "on": -6.059123, + "one": -6.059123, + "or": -5.365976, + "parser": -6.059123, + "percent": -6.059123, + "percent_off": -6.059123, + "plain": -6.059123, + "please": -6.059123, + "positive": -6.059123, + "priority": -6.059123, + "progress": -6.059123, + "provided": -6.059123, + "redeem_by": -6.059123, + "redeemed": -6.059123, + "represents": -6.059123, + "request": -6.059123, + "resource": -4.960511, + "resource.": -6.059123, + "response": -6.059123, + "s": -6.059123, + "section.": -6.059123, + "simplest": -6.059123, + "single": -6.059123, + "stamp": -6.059123, + "state": -4.960511, + "status": -6.059123, + "string": -4.672829, + "such": -6.059123, + "text/plain": -6.059123, + "that": -4.960511, + "the": -3.420066, + "this": -5.365976, + "through": -6.059123, + "to": -4.113213, + "transition": -5.365976, + "transition.": -6.059123, + "true": -6.059123, + "view": -6.059123, + "want": -6.059123, + "we": -6.059123, + "what": -6.059123, + "which": -6.059123, + "will": -5.365976, + "with": -4.960511, + "written": -6.059123, + "you": -5.365976, + "{": -4.672829, + "}": -4.672829, + "–": -5.365976, + }, + "APL": map[string]float64{ + "(": -3.506736, + ")": -3.489037, + "+": -5.047181, + ",": -3.101271, + "-": -5.586178, + "/": -4.893031, + "/C": -7.532088, + "/Functions": -7.532088, + "/Page": -7.532088, + "/Z": -7.532088, + "/config": -7.532088, + "/input": -7.532088, + "/lines": -7.532088, + ":": -3.327396, + ";": -3.327396, + "": -7.532088, + "": -7.532088, + "": -7.532088, + "": -7.532088, + "": -6.838941, + "": -7.532088, + "": -7.532088, + "": -7.532088, + "
":           -7.532088,
+			"":           -8.083020,
+			"":           -5.598113,
+			"":           -8.083020,
+			"":              -4.750815,
+			"":            -6.291260,
+			"":           -6.984408,
+			"":              -5.443963,
+			"":                -8.083020,
+			"
":                -8.083020,
+			"":                -6.003578,
+			"":             -6.473582,
+			"
":              -6.984408,
+			"