Squashed 'go-enry/' content from commit 7e3a9a7

git-subtree-dir: go-enry
git-subtree-split: 7e3a9a7241

internal/code-generator/assets/alias.go.tmpl

@@ -0,0 +1,35 @@
+package data
+
+import "strings"
+
+// LanguageByAliasMap keeps alias for different languages and use the name of the languages as an alias too.
+// All the keys (alias or not) are written in lower case and the whitespaces has been replaced by underscores.
+var LanguageByAliasMap = map[string]string{
+	{{range $alias, $language := . -}}
+		"{{ $alias }}":	{{ printf "%q" $language -}},
+	{{end -}}
+}
+
+// LanguageByAlias looks up the language name by it's alias or name.
+// It mirrors the logic of github linguist and is needed e.g for heuristcs.yml
+// that mixes names and aliases in a language field (see XPM example).
+func LanguageByAlias(langOrAlias string) (lang string, ok bool) {
+	k := convertToAliasKey(langOrAlias)
+	lang, ok = LanguageByAliasMap[k]
+	return
+}
+
+
+// convertToAliasKey converts language name to a key in LanguageByAliasMap.
+// Following
+//   - internal.code-generator.generator.convertToAliasKey()
+//   - GetLanguageByAlias()
+//
+// conventions.
+// It is here to avoid dependency on "generate" and "enry" packages.
+func convertToAliasKey(langName string) string {
+	ak := strings.SplitN(langName, `,`, 2)[0]
+	ak = strings.Replace(ak, ` `, `_`, -1)
+	ak = strings.ToLower(ak)
+	return ak
+}

internal/code-generator/assets/colors.go.tmpl

@@ -0,0 +1,7 @@
+package data
+
+var LanguagesColor = map[string]string{
+  {{range $language, $color := . -}}
+  "{{$language}}": "{{$color -}}",
+  {{end -}}
+}

internal/code-generator/assets/commit.go.tmpl

@@ -0,0 +1,4 @@
+package data
+
+// linguist's commit from which files were generated.
+var LinguistCommit = "{{- getCommit -}}"

internal/code-generator/assets/content.go.tmpl

@@ -0,0 +1,58 @@
+package data
+
+import (
+	"github.com/go-enry/go-enry/v2/data/rule"
+	"github.com/go-enry/go-enry/v2/regex"
+)
+
+var ContentHeuristics = map[string]*Heuristics{
+	{{ range $ext, $rules := . -}}
+		{{ printf "%q" $ext }}: &Heuristics{
+			{{ range $rule := $rules -}}
+				{{template "Rule" $rule}}
+			{{ end -}}
+		},
+	{{ end -}}
+}
+
+{{ define "Rule" -}}
+	{{ if eq .Op "And" -}}
+		rule.And(
+			{{ template "Languages" .Langs -}}
+			{{ range $rule := .Rules -}}
+				{{template "Rule" $rule}}
+			{{ end -}}
+		),
+	{{- else if eq .Op "Or" -}}
+		rule.Or(
+			{{ template "Languages" .Langs -}}
+			{{ template "mustCompile" . }}
+		),
+	{{- else if eq .Op "Not" -}}
+		rule.Not(
+			{{ template "Languages" .Langs -}}
+			{{ template "mustCompile" . }}
+		),
+	{{- else if eq .Op "Always" -}}
+		rule.Always(
+			{{ template "Languages" .Langs -}}
+		),
+	{{ end -}}
+{{ end -}}
+
+
+{{define "Languages" -}}
+	{{with . -}}
+		rule.MatchingLanguages( {{range .}} {{printf "\"%s\"" .}}, {{end}} ),
+	{{ else -}}
+		rule.MatchingLanguages(""),
+	{{end -}}
+{{end}}
+
+{{define "mustCompile" -}}
+	{{ if .IsRE2  -}}
+		regex.MustCompileMultiline({{ .Pattern | stringVal }}),
+	{{- else -}}
+		regex.MustCompileRuby({{ .Pattern | stringVal }}),
+	{{ end -}}
+{{end}}

internal/code-generator/assets/documentation.go.tmpl

@@ -0,0 +1,10 @@
+package data
+
+import "github.com/go-enry/go-enry/v2/regex"
+
+var DocumentationMatchers = []regex.EnryRegexp{
+	{{range $regexp := . -}}
+	regex.MustCompile(`{{ $regexp }}`),
+	{{end -}}
+}
+

internal/code-generator/assets/extension.go.tmpl

@@ -0,0 +1,13 @@
+package data
+
+var LanguagesByExtension = map[string][]string{
+	{{range $extension, $languages := .LanguagesByExtension -}}
+	"{{ $extension }}":	{ {{- $languages | formatStringSlice -}} },
+	{{end -}}
+}
+
+var ExtensionsByLanguage = map[string][]string{
+	{{range $language, $extensions := .ExtensionsByLanguage -}}
+	"{{ $language }}":	{ {{- $extensions | formatStringSlice -}} },
+	{{end -}}
+}

internal/code-generator/assets/filename.go.tmpl

@@ -0,0 +1,7 @@
+package data
+
+var LanguagesByFilename = map[string][]string{
+	{{range $filename, $languages := . -}}
+	"{{ $filename }}":	{ {{- formatStringSlice $languages -}} },
+	{{end -}}
+}

internal/code-generator/assets/frequencies.go.tmpl

@@ -0,0 +1,20 @@
+package data
+
+var LanguagesLogProbabilities = map[string]float64{
+	{{ $freqs := . -}}
+	{{range $index, $language := orderKeys .Languages -}}
+		"{{ $language }}":	{{ languageLogProbability $language -}},
+	{{end -}}
+}
+
+var TokensLogProbabilities = map[string]map[string]float64{
+	{{range $index, $language := orderMapMapKeys .Tokens -}}
+		"{{ $language }}": map[string]float64{
+		{{range $i, $token := index $freqs.Tokens $language | orderKeys -}}
+			{{ quote $token }}: {{ tokenLogProbability $language $token }},
+		{{end -}}
+		},
+	{{end -}}
+}
+
+var TokensTotal = {{ toFloat64 .TokensTotal -}}

internal/code-generator/assets/groups.go.tmpl

@@ -0,0 +1,7 @@
+package data
+
+var LanguagesGroup = map[string]string{
+  {{range $language, $group := . -}}
+  "{{$language}}": "{{$group -}}",
+  {{end -}}
+}

internal/code-generator/assets/header.go.tmpl

@@ -0,0 +1,3 @@
+// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
+// Extracted from github/linguist commit: {{ getCommit }}
+

internal/code-generator/assets/id.go.tmpl

@@ -0,0 +1,7 @@
+package data
+
+var IDByLanguage = map[string]int{
+  {{range $language, $id := . -}}
+  "{{$language}}": {{$id -}},
+  {{end -}}
+}

internal/code-generator/assets/interpreter.go.tmpl

@@ -0,0 +1,7 @@
+package data
+
+var LanguagesByInterpreter = map[string][]string{
+	{{range $interpreter, $languages := . -}}
+	"{{ $interpreter }}":	{ {{- $languages | formatStringSlice -}} },
+	{{end -}}
+}

internal/code-generator/assets/languageInfo.go.tmpl

@@ -0,0 +1,75 @@
+package data
+
+// LanguageInfo exposes the data for a language's Linguist YAML entry as a Go struct.
+// See https://github.com/github/linguist/blob/master/lib/linguist/languages.yml
+type LanguageInfo struct {
+  // Name is the language name. May contain symbols not safe for use in some filesystems (e.g., `F*`).
+  Name string
+  // FSName is the filesystem safe name. Will only be set if Name is not safe for use in all filesystems.
+  FSName string
+  // Type is the language Type. See data.Type for values.
+  Type Type
+  // Color is the CSS hex color to represent the language. Only used if type is "programming" or "markup".
+  Color string
+  // Group is the name of the parent language. Languages in a group are counted in the statistics as the parent language.
+  Group string
+  // Aliases is a slice of additional aliases (implicitly includes name.downcase)
+  Aliases []string
+  // Extensions is a slice of associated extensions (the first one is considered the primary extension).
+  Extensions []string
+  // A slice of associated interpreters
+  Interpreters []string
+  // Filenames is a slice of filenames commonly associated with the language.
+  Filenames []string
+  // MimeType (maps to codemirror_mime_type in linguist.yaml) is the string name of the file mime type used for highlighting whenever a file is edited.
+  MimeType string
+  // TMScope is the TextMate scope that represents this programming language.
+  TMScope string
+  // AceMode is the name of the Ace Mode used for highlighting whenever a file is edited.
+  AceMode string
+  // CodeMirrorMode is the name of the CodeMirror Mode used for highlighting whenever a file is edited.
+  CodeMirrorMode string
+  // Wrap is a boolean flag to enable line wrapping in an editor.
+  Wrap bool
+  // LanguageID is the Linguist-assigned numeric ID for the language.
+  LanguageID int
+}
+
+// LanguageInfoByID allows accessing LanguageInfo by a language's ID.
+var LanguageInfoByID = map[int]LanguageInfo{
+  {{range $language, $info := . -}}
+  {{$info.LanguageID}}: LanguageInfo{
+    Name: "{{$language}}",
+    FSName: "{{$info.FSName}}",
+    Type: TypeForString("{{$info.Type}}"),
+    Color: "{{$info.Color}}",
+    Group: "{{$info.Group}}",
+    Aliases: []string{
+      {{range $alias := $info.Aliases -}}
+      "{{$alias}}",
+      {{end -}}
+    },
+    Extensions: []string{
+      {{range $extension := $info.Extensions -}}
+      "{{$extension}}",
+      {{end -}}
+    },
+    Interpreters: []string{
+      {{range $interpreter := $info.Interpreters -}}
+      "{{$interpreter}}",
+      {{end -}}
+    },
+    Filenames: []string{
+      {{range $filename := $info.Filenames -}}
+      "{{$filename}}",
+      {{end -}}
+    },
+    MimeType: "{{$info.MimeType}}",
+    TMScope: "{{$info.TMScope}}",
+    AceMode: "{{$info.AceMode}}",
+    CodeMirrorMode: "{{$info.CodeMirrorMode}}",
+    Wrap: {{$info.Wrap}},
+    LanguageID: {{$info.LanguageID}},
+  },
+  {{end -}}
+}

internal/code-generator/assets/mimeType.go.tmpl

@@ -0,0 +1,7 @@
+package data
+
+var LanguagesMime = map[string]string{
+  {{range $language, $mime := . -}}
+  "{{$language}}": "{{$mime -}}",
+  {{end -}}
+}

internal/code-generator/assets/type.go.tmpl

@@ -0,0 +1,49 @@
+package data
+
+// Type represent language's type. Either data, programming, markup, prose, or unknown.
+type Type int
+
+// Type's values.
+const (
+	TypeUnknown Type = iota
+	TypeData
+	TypeProgramming
+	TypeMarkup
+	TypeProse
+)
+
+func (t Type) String() string {
+  switch t {
+  case TypeData:
+    return "data"
+  case TypeProgramming:
+    return "programming"
+  case TypeMarkup:
+    return "markup"
+  case TypeProse:
+    return "prose"
+  default:
+    return "unknown"
+  }
+}
+
+func TypeForString(s string) Type {
+  switch s {
+  case "data":
+    return TypeData
+  case "programming":
+    return TypeProgramming
+  case "markup":
+    return TypeMarkup
+  case "prose":
+    return TypeProse
+  default:
+    return TypeUnknown
+  }
+}
+
+var LanguagesType = map[string]int{
+	{{range $language, $type := . -}}
+	"{{ $language }}": {{ $type -}},
+	{{end -}}
+}

internal/code-generator/assets/vendor.go.tmpl

@@ -0,0 +1,22 @@
+package data
+
+import "github.com/go-enry/go-enry/v2/regex"
+
+{{define "mustCompile" -}}
+	{{ if isRE2 .  -}}
+		regex.MustCompile({{ . | stringVal }})
+	{{- else -}}
+		regex.MustCompileRuby({{ . | stringVal }})
+	{{- end -}}
+{{end}}
+
+var VendorMatchers = []regex.EnryRegexp{
+	{{range $re := . -}}
+		{{ template "mustCompile" $re }},
+	{{end -}}
+}
+
+// FastVendorMatcher is equivalent to matching any of the VendorMatchers.
+{{with $singleRE := collateAllRegexps . -}}
+var FastVendorMatcher = {{template "mustCompile" $singleRE}}
+{{end}}