ralsina/tartrazine
1
0
Fork 0
mirror of https://github.com/ralsina/tartrazine.git synced 2024-11-13 23:12:24 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #205 from kuba--/fix-langs

Browse Source 
Do not return empty lang.
This commit is contained in:
Alexander 2019-03-15 18:51:55 +01:00 committed by GitHub
commit 6526da77b3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 26 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -6,4 +6,5 @@ Makefile.main
.idea
.docsrv-resources
build/
vendor/
java/lib/

25
classifier.go
View File

@ -26,9 +26,6 @@ type scoredLanguage struct {
// Classify returns a sorted slice of possible languages sorted by decreasing language's probability
func (c *classifier) Classify(content []byte, candidates map[string]float64) []string {
if len(content) == 0 {
return nil
}
var languages map[string]float64
if len(candidates) == 0 {
@ -44,15 +41,23 @@ func (c *classifier) Classify(content []byte, candidates map[string]float64) []s
}
}
tokens := tokenizer.Tokenize(content)
empty := len(content) == 0
scoredLangs := make([]*scoredLanguage, 0, len(languages))
for language := range languages {
scoredLang := &scoredLanguage{
language: language,
score: c.tokensLogProbability(tokens, language) + c.languagesLogProbabilities[language],
}
scoredLangs = append(scoredLangs, scoredLang)
var tokens []string
if !empty {
tokens = tokenizer.Tokenize(content)
}
for language := range languages {
score := c.languagesLogProbabilities[language]
if !empty {
score += c.tokensLogProbability(tokens, language)
}
scoredLangs = append(scoredLangs, &scoredLanguage{
language: language,
score: score,
})
}
return sortLanguagesByScore(scoredLangs)

9
common.go
View File

@ -41,11 +41,12 @@ func GetLanguage(filename string, content []byte) (language string) {
}
func firstLanguage(languages []string) string {
if len(languages) == 0 {
return OtherLanguage
for _, l := range languages {
if l != "" {
return l
}
}
return languages[0]
return OtherLanguage
}
// GetLanguageByModeline returns detected language. If there are more than one possibles languages

8
common_test.go
View File

@ -101,9 +101,10 @@ func (s *EnryTestSuite) TestGetLanguage() {
expected string
safe bool
}{
{name: "TestGetLanguage_0", filename: "foo.h", content: []byte{}, expected: "C"},
{name: "TestGetLanguage_1", filename: "foo.py", content: []byte{}, expected: "Python"},
{name: "TestGetLanguage_2", filename: "foo.m", content: []byte(":- module"), expected: "Mercury"},
{name: "TestGetLanguage_3", filename: "foo.m", content: nil, expected: OtherLanguage},
{name: "TestGetLanguage_3", filename: "foo.m", content: nil, expected: "MATLAB"},
{name: "TestGetLanguage_4", filename: "foo.mo", content: []byte{0xDE, 0x12, 0x04, 0x95, 0x00, 0x00, 0x00, 0x00}, expected: OtherLanguage},
{name: "TestGetLanguage_5", filename: "", content: nil, expected: OtherLanguage},
}
@ -276,6 +277,7 @@ func (s *EnryTestSuite) TestGetLanguagesByExtension() {
candidates []string
expected []string
}{
{name: "TestGetLanguagesByExtension_0", filename: "foo.h", expected: []string{"C", "C++", "Objective-C"}},
{name: "TestGetLanguagesByExtension_1", filename: "foo.foo", expected: nil},
{name: "TestGetLanguagesByExtension_2", filename: "foo.go", expected: []string{"Go"}},
{name: "TestGetLanguagesByExtension_3", filename: "foo.go.php", expected: []string{"Hack", "PHP"}},
@ -301,7 +303,7 @@ func (s *EnryTestSuite) TestGetLanguagesByClassifier() {
{name: "TestGetLanguagesByClassifier_4", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"python", "ruby", "c++"}, expected: "C++"},
{name: "TestGetLanguagesByClassifier_5", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"ruby"}, expected: "Ruby"},
{name: "TestGetLanguagesByClassifier_6", filename: filepath.Join(s.samplesDir, "Python/django-models-base.py"), candidates: []string{"python", "ruby", "c", "c++"}, expected: "Python"},
{name: "TestGetLanguagesByClassifier_7", filename: "", candidates: []string{"python"}, expected: OtherLanguage},
{name: "TestGetLanguagesByClassifier_7", filename: "", candidates: []string{"python"}, expected: "Python"},
}
for _, test := range test {
@ -339,7 +341,7 @@ func (s *EnryTestSuite) TestGetLanguagesBySpecificClassifier() {
{name: "TestGetLanguagesByClassifier_4", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"python", "ruby", "c++"}, classifier: DefaultClassifier, expected: "C++"},
{name: "TestGetLanguagesByClassifier_5", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"ruby"}, classifier: DefaultClassifier, expected: "Ruby"},
{name: "TestGetLanguagesByClassifier_6", filename: filepath.Join(s.samplesDir, "Python/django-models-base.py"), candidates: []string{"python", "ruby", "c", "c++"}, classifier: DefaultClassifier, expected: "Python"},
{name: "TestGetLanguagesByClassifier_7", filename: os.DevNull, candidates: nil, classifier: DefaultClassifier, expected: OtherLanguage},
{name: "TestGetLanguagesByClassifier_7", filename: os.DevNull, candidates: nil, classifier: DefaultClassifier, expected: "XML"},
}
for _, test := range test {