ralsina/tartrazine
1
0
Fork 0
mirror of https://github.com/ralsina/tartrazine.git synced 2024-11-10 13:32:24 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #30 from mcarmonaa/improvement-rearrange

Browse Source 
Improvement rearranged code
This commit is contained in:
Máximo Cuadros 2017-06-08 09:01:51 +02:00 committed by GitHub
commit c37fe182d4
54 changed files with 75602 additions and 81872 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

655
alias.go
View File

@ -1,16 +1,647 @@
package slinguist package slinguist
import "strings" // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: b6460f8ed6b249281ada099ca28bd8f1230b8892
// GetLanguageByAlias returns the language related to the given alias and ok set to true, // languagesByAlias keeps alias for different languages and use the name of the languages as an alias too.
// or Otherlanguage and ok set to false otherwise. // All the keys (alias or not) are written in lower case and the whitespaces has been replaced by underscores.
func GetLanguageByAlias(alias string) (lang string, ok bool) { var languagesByAlias = map[string]string{
a := strings.Split(alias, `,`)[0] "1c_enterprise": "1C Enterprise",
a = strings.ToLower(a) "abap": "ABAP",
lang, ok = languagesByAlias[a] "abl": "OpenEdge ABL",
if !ok { "abnf": "ABNF",
lang = OtherLanguage "abuild": "Alpine Abuild",
} "aconf": "ApacheConf",
"actionscript": "ActionScript",
return "actionscript3": "ActionScript",
"actionscript_3": "ActionScript",
"ada": "Ada",
"ada2005": "Ada",
"ada95": "Ada",
"advpl": "xBase",
"afdko": "OpenType Feature File",
"agda": "Agda",
"ags": "AGS Script",
"ags_script": "AGS Script",
"ahk": "AutoHotkey",
"alloy": "Alloy",
"alpine_abuild": "Alpine Abuild",
"ampl": "AMPL",
"ant_build_system": "Ant Build System",
"antlr": "ANTLR",
"apache": "ApacheConf",
"apacheconf": "ApacheConf",
"apex": "Apex",
"api_blueprint": "API Blueprint",
"apkbuild": "Alpine Abuild",
"apl": "APL",
"apollo_guidance_computer": "Apollo Guidance Computer",
"applescript": "AppleScript",
"arc": "Arc",
"arduino": "Arduino",
"arexx": "REXX",
"as3": "ActionScript",
"asciidoc": "AsciiDoc",
"asn.1": "ASN.1",
"asp": "ASP",
"aspectj": "AspectJ",
"aspx": "ASP",
"aspx-vb": "ASP",
"assembly": "Assembly",
"ats": "ATS",
"ats2": "ATS",
"au3": "AutoIt",
"augeas": "Augeas",
"autoconf": "M4Sugar",
"autohotkey": "AutoHotkey",
"autoit": "AutoIt",
"autoit3": "AutoIt",
"autoitscript": "AutoIt",
"awk": "Awk",
"b3d": "BlitzBasic",
"bash": "Shell",
"bash_session": "ShellSession",
"bat": "Batchfile",
"batch": "Batchfile",
"batchfile": "Batchfile",
"befunge": "Befunge",
"bison": "Bison",
"bitbake": "BitBake",
"blade": "Blade",
"blitz3d": "BlitzBasic",
"blitzbasic": "BlitzBasic",
"blitzmax": "BlitzMax",
"blitzplus": "BlitzBasic",
"bluespec": "Bluespec",
"bmax": "BlitzMax",
"boo": "Boo",
"bplus": "BlitzBasic",
"brainfuck": "Brainfuck",
"brightscript": "Brightscript",
"bro": "Bro",
"bsdmake": "Makefile",
"byond": "DM",
"c": "C",
"c#": "C#",
"c++": "C++",
"c++-objdump": "Cpp-ObjDump",
"c-objdump": "C-ObjDump",
"c2hs": "C2hs Haskell",
"c2hs_haskell": "C2hs Haskell",
"cap'n_proto": "Cap'n Proto",
"carto": "CartoCSS",
"cartocss": "CartoCSS",
"ceylon": "Ceylon",
"cfc": "ColdFusion CFC",
"cfm": "ColdFusion",
"cfml": "ColdFusion",
"chapel": "Chapel",
"charity": "Charity",
"chpl": "Chapel",
"chuck": "ChucK",
"cirru": "Cirru",
"clarion": "Clarion",
"clean": "Clean",
"click": "Click",
"clipper": "xBase",
"clips": "CLIPS",
"clojure": "Clojure",
"closure_templates": "Closure Templates",
"cmake": "CMake",
"cobol": "COBOL",
"coffee": "CoffeeScript",
"coffee-script": "CoffeeScript",
"coffeescript": "CoffeeScript",
"coldfusion": "ColdFusion",
"coldfusion_cfc": "ColdFusion CFC",
"coldfusion_html": "ColdFusion",
"collada": "COLLADA",
"common_lisp": "Common Lisp",
"component_pascal": "Component Pascal",
"console": "ShellSession",
"cool": "Cool",
"coq": "Coq",
"cpp": "C++",
"cpp-objdump": "Cpp-ObjDump",
"creole": "Creole",
"crystal": "Crystal",
"csharp": "C#",
"cson": "CSON",
"csound": "Csound",
"csound-csd": "Csound Document",
"csound-orc": "Csound",
"csound-sco": "Csound Score",
"csound_document": "Csound Document",
"csound_score": "Csound Score",
"css": "CSS",
"csv": "CSV",
"cucumber": "Gherkin",
"cuda": "Cuda",
"cweb": "CWeb",
"cycript": "Cycript",
"cython": "Cython",
"d": "D",
"d-objdump": "D-ObjDump",
"darcs_patch": "Darcs Patch",
"dart": "Dart",
"dcl": "DIGITAL Command Language",
"delphi": "Component Pascal",
"desktop": "desktop",
"diff": "Diff",
"digital_command_language": "DIGITAL Command Language",
"django": "HTML+Django",
"dm": "DM",
"dns_zone": "DNS Zone",
"dockerfile": "Dockerfile",
"dogescript": "Dogescript",
"dosbatch": "Batchfile",
"dosini": "INI",
"dpatch": "Darcs Patch",
"dtrace": "DTrace",
"dtrace-script": "DTrace",
"dylan": "Dylan",
"e": "E",
"eagle": "Eagle",
"ebnf": "EBNF",
"ec": "eC",
"ecere_projects": "Ecere Projects",
"ecl": "ECL",
"eclipse": "ECLiPSe",
"ecr": "HTML+ECR",
"edn": "edn",
"eex": "HTML+EEX",
"eiffel": "Eiffel",
"ejs": "EJS",
"elisp": "Emacs Lisp",
"elixir": "Elixir",
"elm": "Elm",
"emacs": "Emacs Lisp",
"emacs_lisp": "Emacs Lisp",
"emberscript": "EmberScript",
"eq": "EQ",
"erb": "HTML+ERB",
"erlang": "Erlang",
"f#": "F#",
"factor": "Factor",
"fancy": "Fancy",
"fantom": "Fantom",
"filebench_wml": "Filebench WML",
"filterscript": "Filterscript",
"fish": "fish",
"flex": "Lex",
"flux": "FLUX",
"formatted": "Formatted",
"forth": "Forth",
"fortran": "Fortran",
"foxpro": "xBase",
"freemarker": "FreeMarker",
"frege": "Frege",
"fsharp": "F#",
"ftl": "FreeMarker",
"fundamental": "Text",
"g-code": "G-code",
"game_maker_language": "Game Maker Language",
"gams": "GAMS",
"gap": "GAP",
"gcc_machine_description": "GCC Machine Description",
"gdb": "GDB",
"gdscript": "GDScript",
"genie": "Genie",
"genshi": "Genshi",
"gentoo_ebuild": "Gentoo Ebuild",
"gentoo_eclass": "Gentoo Eclass",
"gettext_catalog": "Gettext Catalog",
"gf": "Grammatical Framework",
"gherkin": "Gherkin",
"glsl": "GLSL",
"glyph": "Glyph",
"gn": "GN",
"gnuplot": "Gnuplot",
"go": "Go",
"golang": "Go",
"golo": "Golo",
"gosu": "Gosu",
"grace": "Grace",
"gradle": "Gradle",
"grammatical_framework": "Grammatical Framework",
"graph_modeling_language": "Graph Modeling Language",
"graphql": "GraphQL",
"graphviz_(dot)": "Graphviz (DOT)",
"groovy": "Groovy",
"groovy_server_pages": "Groovy Server Pages",
"gsp": "Groovy Server Pages",
"hack": "Hack",
"haml": "Haml",
"handlebars": "Handlebars",
"harbour": "Harbour",
"haskell": "Haskell",
"haxe": "Haxe",
"hbs": "Handlebars",
"hcl": "HCL",
"hlsl": "HLSL",
"html": "HTML",
"html+django": "HTML+Django",
"html+django/jinja": "HTML+Django",
"html+ecr": "HTML+ECR",
"html+eex": "HTML+EEX",
"html+erb": "HTML+ERB",
"html+jinja": "HTML+Django",
"html+php": "HTML+PHP",
"html+ruby": "RHTML",
"htmlbars": "Handlebars",
"htmldjango": "HTML+Django",
"http": "HTTP",
"hy": "Hy",
"hylang": "Hy",
"hyphy": "HyPhy",
"i7": "Inform 7",
"idl": "IDL",
"idris": "Idris",
"igor": "IGOR Pro",
"igor_pro": "IGOR Pro",
"igorpro": "IGOR Pro",
"inc": "PHP",
"inform7": "Inform 7",
"inform_7": "Inform 7",
"ini": "INI",
"inno_setup": "Inno Setup",
"io": "Io",
"ioke": "Ioke",
"ipython_notebook": "Jupyter Notebook",
"irc": "IRC log",
"irc_log": "IRC log",
"irc_logs": "IRC log",
"isabelle": "Isabelle",
"isabelle_root": "Isabelle ROOT",
"j": "J",
"jasmin": "Jasmin",
"java": "Java",
"java_server_page": "Groovy Server Pages",
"java_server_pages": "Java Server Pages",
"javascript": "JavaScript",
"jflex": "JFlex",
"jison": "Jison",
"jison_lex": "Jison Lex",
"jolie": "Jolie",
"jruby": "Ruby",
"js": "JavaScript",
"json": "JSON",
"json5": "JSON5",
"jsoniq": "JSONiq",
"jsonld": "JSONLD",
"jsp": "Java Server Pages",
"jsx": "JSX",
"julia": "Julia",
"jupyter_notebook": "Jupyter Notebook",
"kicad": "KiCad",
"kit": "Kit",
"kotlin": "Kotlin",
"krl": "KRL",
"labview": "LabVIEW",
"lasso": "Lasso",
"lassoscript": "Lasso",
"latex": "TeX",
"latte": "Latte",
"lean": "Lean",
"less": "Less",
"lex": "Lex",
"lfe": "LFE",
"lhaskell": "Literate Haskell",
"lhs": "Literate Haskell",
"lilypond": "LilyPond",
"limbo": "Limbo",
"linker_script": "Linker Script",
"linux_kernel_module": "Linux Kernel Module",
"liquid": "Liquid",
"lisp": "Common Lisp",
"litcoffee": "Literate CoffeeScript",
"literate_agda": "Literate Agda",
"literate_coffeescript": "Literate CoffeeScript",
"literate_haskell": "Literate Haskell",
"live-script": "LiveScript",
"livescript": "LiveScript",
"llvm": "LLVM",
"logos": "Logos",
"logtalk": "Logtalk",
"lolcode": "LOLCODE",
"lookml": "LookML",
"loomscript": "LoomScript",
"ls": "LiveScript",
"lsl": "LSL",
"lua": "Lua",
"m": "M",
"m4": "M4",
"m4sugar": "M4Sugar",
"macruby": "Ruby",
"make": "Makefile",
"makefile": "Makefile",
"mako": "Mako",
"markdown": "Markdown",
"marko": "Marko",
"markojs": "Marko",
"mask": "Mask",
"mathematica": "Mathematica",
"matlab": "Matlab",
"maven_pom": "Maven POM",
"max": "Max",
"max/msp": "Max",
"maxmsp": "Max",
"maxscript": "MAXScript",
"mediawiki": "MediaWiki",
"mercury": "Mercury",
"meson": "Meson",
"metal": "Metal",
"mf": "Makefile",
"minid": "MiniD",
"mirah": "Mirah",
"mma": "Mathematica",
"modelica": "Modelica",
"modula-2": "Modula-2",
"module_management_system": "Module Management System",
"monkey": "Monkey",
"moocode": "Moocode",
"moonscript": "MoonScript",
"mql4": "MQL4",
"mql5": "MQL5",
"mtml": "MTML",
"muf": "MUF",
"mumps": "M",
"mupad": "mupad",
"myghty": "Myghty",
"nasm": "Assembly",
"ncl": "NCL",
"nemerle": "Nemerle",
"nesc": "nesC",
"netlinx": "NetLinx",
"netlinx+erb": "NetLinx+ERB",
"netlogo": "NetLogo",
"newlisp": "NewLisp",
"nginx": "Nginx",
"nginx_configuration_file": "Nginx",
"nim": "Nim",
"ninja": "Ninja",
"nit": "Nit",
"nix": "Nix",
"nixos": "Nix",
"njk": "HTML+Django",
"nl": "NL",
"node": "JavaScript",
"nroff": "Roff",
"nsis": "NSIS",
"nu": "Nu",
"numpy": "NumPy",
"nunjucks": "HTML+Django",
"nush": "Nu",
"nvim": "Vim script",
"obj-c": "Objective-C",
"obj-c++": "Objective-C++",
"obj-j": "Objective-J",
"objc": "Objective-C",
"objc++": "Objective-C++",
"objdump": "ObjDump",
"objective-c": "Objective-C",
"objective-c++": "Objective-C++",
"objective-j": "Objective-J",
"objectivec": "Objective-C",
"objectivec++": "Objective-C++",
"objectivej": "Objective-J",
"objectpascal": "Component Pascal",
"objj": "Objective-J",
"ocaml": "OCaml",
"octave": "Matlab",
"omgrofl": "Omgrofl",
"ooc": "ooc",
"opa": "Opa",
"opal": "Opal",
"opencl": "OpenCL",
"openedge": "OpenEdge ABL",
"openedge_abl": "OpenEdge ABL",
"openrc": "OpenRC runscript",
"openrc_runscript": "OpenRC runscript",
"openscad": "OpenSCAD",
"opentype_feature_file": "OpenType Feature File",
"org": "Org",
"osascript": "AppleScript",
"ox": "Ox",
"oxygene": "Oxygene",
"oz": "Oz",
"p4": "P4",
"pan": "Pan",
"pandoc": "Markdown",
"papyrus": "Papyrus",
"parrot": "Parrot",
"parrot_assembly": "Parrot Assembly",
"parrot_internal_representation": "Parrot Internal Representation",
"pascal": "Pascal",
"pasm": "Parrot Assembly",
"pawn": "PAWN",
"pep8": "Pep8",
"perl": "Perl",
"perl6": "Perl6",
"php": "PHP",
"pic": "Pic",
"pickle": "Pickle",
"picolisp": "PicoLisp",
"piglatin": "PigLatin",
"pike": "Pike",
"pir": "Parrot Internal Representation",
"plpgsql": "PLpgSQL",
"plsql": "PLSQL",
"pod": "Pod",
"pogoscript": "PogoScript",
"pony": "Pony",
"posh": "PowerShell",
"postscr": "PostScript",
"postscript": "PostScript",
"pot": "Gettext Catalog",
"pov-ray": "POV-Ray SDL",
"pov-ray_sdl": "POV-Ray SDL",
"povray": "POV-Ray SDL",
"powerbuilder": "PowerBuilder",
"powershell": "PowerShell",
"processing": "Processing",
"progress": "OpenEdge ABL",
"prolog": "Prolog",
"propeller_spin": "Propeller Spin",
"protobuf": "Protocol Buffer",
"protocol_buffer": "Protocol Buffer",
"protocol_buffers": "Protocol Buffer",
"public_key": "Public Key",
"pug": "Pug",
"puppet": "Puppet",
"pure_data": "Pure Data",
"purebasic": "PureBasic",
"purescript": "PureScript",
"pycon": "Python console",
"pyrex": "Cython",
"python": "Python",
"python_console": "Python console",
"python_traceback": "Python traceback",
"qmake": "QMake",
"qml": "QML",
"r": "R",
"racket": "Racket",
"ragel": "Ragel",
"ragel-rb": "Ragel",
"ragel-ruby": "Ragel",
"rake": "Ruby",
"raml": "RAML",
"rascal": "Rascal",
"raw": "Raw token data",
"raw_token_data": "Raw token data",
"rb": "Ruby",
"rbx": "Ruby",
"rdoc": "RDoc",
"realbasic": "REALbasic",
"reason": "Reason",
"rebol": "Rebol",
"red": "Red",
"red/system": "Red",
"redcode": "Redcode",
"regex": "Regular Expression",
"regexp": "Regular Expression",
"regular_expression": "Regular Expression",
"ren'py": "Ren'Py",
"renderscript": "RenderScript",
"renpy": "Ren'Py",
"restructuredtext": "reStructuredText",
"rexx": "REXX",
"rhtml": "RHTML",
"rmarkdown": "RMarkdown",
"robotframework": "RobotFramework",
"roff": "Roff",
"rouge": "Rouge",
"rpm_spec": "RPM Spec",
"rscript": "R",
"rss": "XML",
"rst": "reStructuredText",
"ruby": "Ruby",
"runoff": "RUNOFF",
"rust": "Rust",
"rusthon": "Python",
"sage": "Sage",
"salt": "SaltStack",
"saltstack": "SaltStack",
"saltstate": "SaltStack",
"sas": "SAS",
"sass": "Sass",
"scala": "Scala",
"scaml": "Scaml",
"scheme": "Scheme",
"scilab": "Scilab",
"scss": "SCSS",
"self": "Self",
"sh": "Shell",
"shaderlab": "ShaderLab",
"shell": "Shell",
"shell-script": "Shell",
"shellsession": "ShellSession",
"shen": "Shen",
"slash": "Slash",
"slim": "Slim",
"smali": "Smali",
"smalltalk": "Smalltalk",
"smarty": "Smarty",
"sml": "Standard ML",
"smt": "SMT",
"sourcemod": "SourcePawn",
"sourcepawn": "SourcePawn",
"sparql": "SPARQL",
"specfile": "RPM Spec",
"spline_font_database": "Spline Font Database",
"splus": "R",
"sqf": "SQF",
"sql": "SQL",
"sqlpl": "SQLPL",
"squeak": "Smalltalk",
"squirrel": "Squirrel",
"srecode_template": "SRecode Template",
"stan": "Stan",
"standard_ml": "Standard ML",
"stata": "Stata",
"ston": "STON",
"stylus": "Stylus",
"sublime_text_config": "Sublime Text Config",
"subrip_text": "SubRip Text",
"supercollider": "SuperCollider",
"svg": "SVG",
"swift": "Swift",
"systemverilog": "SystemVerilog",
"tcl": "Tcl",
"tcsh": "Tcsh",
"tea": "Tea",
"terra": "Terra",
"tex": "TeX",
"text": "Text",
"textile": "Textile",
"thrift": "Thrift",
"ti_program": "TI Program",
"tl": "Type Language",
"tla": "TLA",
"toml": "TOML",
"ts": "TypeScript",
"turing": "Turing",
"turtle": "Turtle",
"twig": "Twig",
"txl": "TXL",
"type_language": "Type Language",
"typescript": "TypeScript",
"udiff": "Diff",
"unified_parallel_c": "Unified Parallel C",
"unity3d_asset": "Unity3D Asset",
"unix_assembly": "Unix Assembly",
"uno": "Uno",
"unrealscript": "UnrealScript",
"ur": "UrWeb",
"ur/web": "UrWeb",
"urweb": "UrWeb",
"vala": "Vala",
"vb.net": "Visual Basic",
"vbnet": "Visual Basic",
"vcl": "VCL",
"verilog": "Verilog",
"vhdl": "VHDL",
"vim": "Vim script",
"vim_script": "Vim script",
"viml": "Vim script",
"visual_basic": "Visual Basic",
"volt": "Volt",
"vue": "Vue",
"wasm": "WebAssembly",
"wast": "WebAssembly",
"wavefront_material": "Wavefront Material",
"wavefront_object": "Wavefront Object",
"web_ontology_language": "Web Ontology Language",
"webassembly": "WebAssembly",
"webidl": "WebIDL",
"winbatch": "Batchfile",
"wisp": "wisp",
"world_of_warcraft_addon_data": "World of Warcraft Addon Data",
"wsdl": "XML",
"x10": "X10",
"xbase": "xBase",
"xc": "XC",
"xcompose": "XCompose",
"xhtml": "HTML",
"xml": "XML",
"xml+genshi": "Genshi",
"xml+kid": "Genshi",
"xojo": "Xojo",
"xpages": "XPages",
"xproc": "XProc",
"xquery": "XQuery",
"xs": "XS",
"xsd": "XML",
"xsl": "XSLT",
"xslt": "XSLT",
"xten": "X10",
"xtend": "Xtend",
"yacc": "Yacc",
"yaml": "YAML",
"yang": "YANG",
"yml": "YAML",
"zephir": "Zephir",
"zimpl": "Zimpl",
"zsh": "Shell",
} }

28
alias_test.go
View File

@ -1,28 +0,0 @@
package slinguist
import . "gopkg.in/check.v1"
func (s *TSuite) TestGetLanguageByAlias(c *C) {
tests := []struct {
alias string
expectedLang string
expectedOk bool
}{
{alias: "BestLanguageEver", expectedLang: OtherLanguage, expectedOk: false},
{alias: "aspx-vb", expectedLang: "ASP", expectedOk: true},
{alias: "C++", expectedLang: "C++", expectedOk: true},
{alias: "c++", expectedLang: "C++", expectedOk: true},
{alias: "objc", expectedLang: "Objective-C", expectedOk: true},
{alias: "golang", expectedLang: "Go", expectedOk: true},
{alias: "GOLANG", expectedLang: "Go", expectedOk: true},
{alias: "bsdmake", expectedLang: "Makefile", expectedOk: true},
{alias: "xhTmL", expectedLang: "HTML", expectedOk: true},
{alias: "python", expectedLang: "Python", expectedOk: true},
}
for _, test := range tests {
lang, ok := GetLanguageByAlias(test.alias)
c.Assert(lang, Equals, test.expectedLang)
c.Assert(ok, Equals, test.expectedOk)
}
}

644
aliases_map.go
View File

@ -1,644 +0,0 @@
package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: 60f864a138650dd17fafc94814be9ee2d3aaef8c
// languagesByAlias keeps alias for different languages and use the name of the languages as a alias too. All the
// keys (alias or not) are written in lower case and the whitespaces has been replaced by underscores.
var languagesByAlias = map[string]string{
"1c_enterprise": "1C Enterprise",
"abap": "ABAP",
"abl": "OpenEdge ABL",
"abnf": "ABNF",
"abuild": "Alpine Abuild",
"aconf": "ApacheConf",
"actionscript": "ActionScript",
"actionscript3": "ActionScript",
"actionscript_3": "ActionScript",
"ada": "Ada",
"ada2005": "Ada",
"ada95": "Ada",
"advpl": "xBase",
"afdko": "OpenType Feature File",
"agda": "Agda",
"ags": "AGS Script",
"ags_script": "AGS Script",
"ahk": "AutoHotkey",
"alloy": "Alloy",
"alpine_abuild": "Alpine Abuild",
"ampl": "AMPL",
"ant_build_system": "Ant Build System",
"antlr": "ANTLR",
"apache": "ApacheConf",
"apacheconf": "ApacheConf",
"apex": "Apex",
"api_blueprint": "API Blueprint",
"apkbuild": "Alpine Abuild",
"apl": "APL",
"apollo_guidance_computer": "Apollo Guidance Computer",
"applescript": "AppleScript",
"arc": "Arc",
"arduino": "Arduino",
"arexx": "REXX",
"as3": "ActionScript",
"asciidoc": "AsciiDoc",
"asn.1": "ASN.1",
"asp": "ASP",
"aspectj": "AspectJ",
"aspx": "ASP",
"aspx-vb": "ASP",
"assembly": "Assembly",
"ats": "ATS",
"ats2": "ATS",
"au3": "AutoIt",
"augeas": "Augeas",
"autoconf": "M4Sugar",
"autohotkey": "AutoHotkey",
"autoit": "AutoIt",
"autoit3": "AutoIt",
"autoitscript": "AutoIt",
"awk": "Awk",
"b3d": "BlitzBasic",
"bash": "Shell",
"bash_session": "ShellSession",
"bat": "Batchfile",
"batch": "Batchfile",
"batchfile": "Batchfile",
"befunge": "Befunge",
"bison": "Bison",
"bitbake": "BitBake",
"blade": "Blade",
"blitz3d": "BlitzBasic",
"blitzbasic": "BlitzBasic",
"blitzmax": "BlitzMax",
"blitzplus": "BlitzBasic",
"bluespec": "Bluespec",
"bmax": "BlitzMax",
"boo": "Boo",
"bplus": "BlitzBasic",
"brainfuck": "Brainfuck",
"brightscript": "Brightscript",
"bro": "Bro",
"bsdmake": "Makefile",
"byond": "DM",
"c": "C",
"c#": "C#",
"c++": "C++",
"c++-objdump": "Cpp-ObjDump",
"c-objdump": "C-ObjDump",
"c2hs": "C2hs Haskell",
"c2hs_haskell": "C2hs Haskell",
"cap'n_proto": "Cap'n Proto",
"carto": "CartoCSS",
"cartocss": "CartoCSS",
"ceylon": "Ceylon",
"cfc": "ColdFusion CFC",
"cfm": "ColdFusion",
"cfml": "ColdFusion",
"chapel": "Chapel",
"charity": "Charity",
"chpl": "Chapel",
"chuck": "ChucK",
"cirru": "Cirru",
"clarion": "Clarion",
"clean": "Clean",
"click": "Click",
"clipper": "xBase",
"clips": "CLIPS",
"clojure": "Clojure",
"closure_templates": "Closure Templates",
"cmake": "CMake",
"cobol": "COBOL",
"coffee": "CoffeeScript",
"coffee-script": "CoffeeScript",
"coffeescript": "CoffeeScript",
"coldfusion": "ColdFusion",
"coldfusion_cfc": "ColdFusion CFC",
"coldfusion_html": "ColdFusion",
"collada": "COLLADA",
"common_lisp": "Common Lisp",
"component_pascal": "Component Pascal",
"console": "ShellSession",
"cool": "Cool",
"coq": "Coq",
"cpp": "C++",
"cpp-objdump": "Cpp-ObjDump",
"creole": "Creole",
"crystal": "Crystal",
"csharp": "C#",
"cson": "CSON",
"csound": "Csound",
"csound-csd": "Csound Document",
"csound-orc": "Csound",
"csound-sco": "Csound Score",
"csound_document": "Csound Document",
"csound_score": "Csound Score",
"css": "CSS",
"csv": "CSV",
"cucumber": "Gherkin",
"cuda": "Cuda",
"cweb": "CWeb",
"cycript": "Cycript",
"cython": "Cython",
"d": "D",
"d-objdump": "D-ObjDump",
"darcs_patch": "Darcs Patch",
"dart": "Dart",
"dcl": "DIGITAL Command Language",
"delphi": "Component Pascal",
"desktop": "desktop",
"diff": "Diff",
"digital_command_language": "DIGITAL Command Language",
"django": "HTML+Django",
"dm": "DM",
"dns_zone": "DNS Zone",
"dockerfile": "Dockerfile",
"dogescript": "Dogescript",
"dosbatch": "Batchfile",
"dosini": "INI",
"dpatch": "Darcs Patch",
"dtrace": "DTrace",
"dtrace-script": "DTrace",
"dylan": "Dylan",
"e": "E",
"eagle": "Eagle",
"ebnf": "EBNF",
"ec": "eC",
"ecere_projects": "Ecere Projects",
"ecl": "ECL",
"eclipse": "ECLiPSe",
"ecr": "HTML+ECR",
"edn": "edn",
"eex": "HTML+EEX",
"eiffel": "Eiffel",
"ejs": "EJS",
"elisp": "Emacs Lisp",
"elixir": "Elixir",
"elm": "Elm",
"emacs": "Emacs Lisp",
"emacs_lisp": "Emacs Lisp",
"emberscript": "EmberScript",
"eq": "EQ",
"erb": "HTML+ERB",
"erlang": "Erlang",
"f#": "F#",
"factor": "Factor",
"fancy": "Fancy",
"fantom": "Fantom",
"filebench_wml": "Filebench WML",
"filterscript": "Filterscript",
"fish": "fish",
"flex": "Lex",
"flux": "FLUX",
"formatted": "Formatted",
"forth": "Forth",
"fortran": "Fortran",
"foxpro": "xBase",
"freemarker": "FreeMarker",
"frege": "Frege",
"fsharp": "F#",
"ftl": "FreeMarker",
"fundamental": "Text",
"g-code": "G-code",
"game_maker_language": "Game Maker Language",
"gams": "GAMS",
"gap": "GAP",
"gcc_machine_description": "GCC Machine Description",
"gdb": "GDB",
"gdscript": "GDScript",
"genie": "Genie",
"genshi": "Genshi",
"gentoo_ebuild": "Gentoo Ebuild",
"gentoo_eclass": "Gentoo Eclass",
"gettext_catalog": "Gettext Catalog",
"gf": "Grammatical Framework",
"gherkin": "Gherkin",
"glsl": "GLSL",
"glyph": "Glyph",
"gn": "GN",
"gnuplot": "Gnuplot",
"go": "Go",
"golang": "Go",
"golo": "Golo",
"gosu": "Gosu",
"grace": "Grace",
"gradle": "Gradle",
"grammatical_framework": "Grammatical Framework",
"graph_modeling_language": "Graph Modeling Language",
"graphql": "GraphQL",
"graphviz_(dot)": "Graphviz (DOT)",
"groovy": "Groovy",
"groovy_server_pages": "Groovy Server Pages",
"gsp": "Groovy Server Pages",
"hack": "Hack",
"haml": "Haml",
"handlebars": "Handlebars",
"harbour": "Harbour",
"haskell": "Haskell",
"haxe": "Haxe",
"hbs": "Handlebars",
"hcl": "HCL",
"hlsl": "HLSL",
"html": "HTML",
"html+django": "HTML+Django",
"html+django/jinja": "HTML+Django",
"html+ecr": "HTML+ECR",
"html+eex": "HTML+EEX",
"html+erb": "HTML+ERB",
"html+jinja": "HTML+Django",
"html+php": "HTML+PHP",
"html+ruby": "RHTML",
"htmlbars": "Handlebars",
"htmldjango": "HTML+Django",
"http": "HTTP",
"hy": "Hy",
"hylang": "Hy",
"hyphy": "HyPhy",
"i7": "Inform 7",
"idl": "IDL",
"idris": "Idris",
"igor": "IGOR Pro",
"igor_pro": "IGOR Pro",
"igorpro": "IGOR Pro",
"inc": "PHP",
"inform7": "Inform 7",
"inform_7": "Inform 7",
"ini": "INI",
"inno_setup": "Inno Setup",
"io": "Io",
"ioke": "Ioke",
"ipython_notebook": "Jupyter Notebook",
"irc": "IRC log",
"irc_log": "IRC log",
"irc_logs": "IRC log",
"isabelle": "Isabelle",
"isabelle_root": "Isabelle ROOT",
"j": "J",
"jasmin": "Jasmin",
"java": "Java",
"java_server_page": "Groovy Server Pages",
"java_server_pages": "Java Server Pages",
"javascript": "JavaScript",
"jflex": "JFlex",
"jison": "Jison",
"jison_lex": "Jison Lex",
"jolie": "Jolie",
"jruby": "Ruby",
"js": "JavaScript",
"json": "JSON",
"json5": "JSON5",
"jsoniq": "JSONiq",
"jsonld": "JSONLD",
"jsp": "Java Server Pages",
"jsx": "JSX",
"julia": "Julia",
"jupyter_notebook": "Jupyter Notebook",
"kicad": "KiCad",
"kit": "Kit",
"kotlin": "Kotlin",
"krl": "KRL",
"labview": "LabVIEW",
"lasso": "Lasso",
"lassoscript": "Lasso",
"latex": "TeX",
"latte": "Latte",
"lean": "Lean",
"less": "Less",
"lex": "Lex",
"lfe": "LFE",
"lhaskell": "Literate Haskell",
"lhs": "Literate Haskell",
"lilypond": "LilyPond",
"limbo": "Limbo",
"linker_script": "Linker Script",
"linux_kernel_module": "Linux Kernel Module",
"liquid": "Liquid",
"lisp": "Common Lisp",
"litcoffee": "Literate CoffeeScript",
"literate_agda": "Literate Agda",
"literate_coffeescript": "Literate CoffeeScript",
"literate_haskell": "Literate Haskell",
"live-script": "LiveScript",
"livescript": "LiveScript",
"llvm": "LLVM",
"logos": "Logos",
"logtalk": "Logtalk",
"lolcode": "LOLCODE",
"lookml": "LookML",
"loomscript": "LoomScript",
"ls": "LiveScript",
"lsl": "LSL",
"lua": "Lua",
"m": "M",
"m4": "M4",
"m4sugar": "M4Sugar",
"macruby": "Ruby",
"make": "Makefile",
"makefile": "Makefile",
"mako": "Mako",
"markdown": "Markdown",
"marko": "Marko",
"markojs": "Marko",
"mask": "Mask",
"mathematica": "Mathematica",
"matlab": "Matlab",
"maven_pom": "Maven POM",
"max": "Max",
"max/msp": "Max",
"maxmsp": "Max",
"maxscript": "MAXScript",
"mediawiki": "MediaWiki",
"mercury": "Mercury",
"meson": "Meson",
"metal": "Metal",
"mf": "Makefile",
"minid": "MiniD",
"mirah": "Mirah",
"mma": "Mathematica",
"modelica": "Modelica",
"modula-2": "Modula-2",
"module_management_system": "Module Management System",
"monkey": "Monkey",
"moocode": "Moocode",
"moonscript": "MoonScript",
"mql4": "MQL4",
"mql5": "MQL5",
"mtml": "MTML",
"muf": "MUF",
"mumps": "M",
"mupad": "mupad",
"myghty": "Myghty",
"nasm": "Assembly",
"ncl": "NCL",
"nemerle": "Nemerle",
"nesc": "nesC",
"netlinx": "NetLinx",
"netlinx+erb": "NetLinx+ERB",
"netlogo": "NetLogo",
"newlisp": "NewLisp",
"nginx": "Nginx",
"nginx_configuration_file": "Nginx",
"nim": "Nim",
"ninja": "Ninja",
"nit": "Nit",
"nix": "Nix",
"nixos": "Nix",
"njk": "HTML+Django",
"nl": "NL",
"node": "JavaScript",
"nroff": "Roff",
"nsis": "NSIS",
"nu": "Nu",
"numpy": "NumPy",
"nunjucks": "HTML+Django",
"nush": "Nu",
"nvim": "Vim script",
"obj-c": "Objective-C",
"obj-c++": "Objective-C++",
"obj-j": "Objective-J",
"objc": "Objective-C",
"objc++": "Objective-C++",
"objdump": "ObjDump",
"objective-c": "Objective-C",
"objective-c++": "Objective-C++",
"objective-j": "Objective-J",
"objectivec": "Objective-C",
"objectivec++": "Objective-C++",
"objectivej": "Objective-J",
"objectpascal": "Component Pascal",
"objj": "Objective-J",
"ocaml": "OCaml",
"octave": "Matlab",
"omgrofl": "Omgrofl",
"ooc": "ooc",
"opa": "Opa",
"opal": "Opal",
"opencl": "OpenCL",
"openedge": "OpenEdge ABL",
"openedge_abl": "OpenEdge ABL",
"openrc": "OpenRC runscript",
"openrc_runscript": "OpenRC runscript",
"openscad": "OpenSCAD",
"opentype_feature_file": "OpenType Feature File",
"org": "Org",
"osascript": "AppleScript",
"ox": "Ox",
"oxygene": "Oxygene",
"oz": "Oz",
"p4": "P4",
"pan": "Pan",
"pandoc": "Markdown",
"papyrus": "Papyrus",
"parrot": "Parrot",
"parrot_assembly": "Parrot Assembly",
"parrot_internal_representation": "Parrot Internal Representation",
"pascal": "Pascal",
"pasm": "Parrot Assembly",
"pawn": "PAWN",
"pep8": "Pep8",
"perl": "Perl",
"perl6": "Perl6",
"php": "PHP",
"pic": "Pic",
"pickle": "Pickle",
"picolisp": "PicoLisp",
"piglatin": "PigLatin",
"pike": "Pike",
"pir": "Parrot Internal Representation",
"plpgsql": "PLpgSQL",
"plsql": "PLSQL",
"pod": "Pod",
"pogoscript": "PogoScript",
"pony": "Pony",
"posh": "PowerShell",
"postscr": "PostScript",
"postscript": "PostScript",
"pot": "Gettext Catalog",
"pov-ray": "POV-Ray SDL",
"pov-ray_sdl": "POV-Ray SDL",
"povray": "POV-Ray SDL",
"powerbuilder": "PowerBuilder",
"powershell": "PowerShell",
"processing": "Processing",
"progress": "OpenEdge ABL",
"prolog": "Prolog",
"propeller_spin": "Propeller Spin",
"protobuf": "Protocol Buffer",
"protocol_buffer": "Protocol Buffer",
"protocol_buffers": "Protocol Buffer",
"public_key": "Public Key",
"pug": "Pug",
"puppet": "Puppet",
"pure_data": "Pure Data",
"purebasic": "PureBasic",
"purescript": "PureScript",
"pycon": "Python console",
"pyrex": "Cython",
"python": "Python",
"python_console": "Python console",
"python_traceback": "Python traceback",
"qmake": "QMake",
"qml": "QML",
"r": "R",
"racket": "Racket",
"ragel": "Ragel",
"ragel-rb": "Ragel",
"ragel-ruby": "Ragel",
"rake": "Ruby",
"raml": "RAML",
"rascal": "Rascal",
"raw": "Raw token data",
"raw_token_data": "Raw token data",
"rb": "Ruby",
"rbx": "Ruby",
"rdoc": "RDoc",
"realbasic": "REALbasic",
"reason": "Reason",
"rebol": "Rebol",
"red": "Red",
"red/system": "Red",
"redcode": "Redcode",
"regex": "Regular Expression",
"regexp": "Regular Expression",
"regular_expression": "Regular Expression",
"ren'py": "Ren'Py",
"renderscript": "RenderScript",
"renpy": "Ren'Py",
"restructuredtext": "reStructuredText",
"rexx": "REXX",
"rhtml": "RHTML",
"rmarkdown": "RMarkdown",
"robotframework": "RobotFramework",
"roff": "Roff",
"rouge": "Rouge",
"rpm_spec": "RPM Spec",
"rscript": "R",
"rss": "XML",
"rst": "reStructuredText",
"ruby": "Ruby",
"runoff": "RUNOFF",
"rust": "Rust",
"rusthon": "Python",
"sage": "Sage",
"salt": "SaltStack",
"saltstack": "SaltStack",
"saltstate": "SaltStack",
"sas": "SAS",
"sass": "Sass",
"scala": "Scala",
"scaml": "Scaml",
"scheme": "Scheme",
"scilab": "Scilab",
"scss": "SCSS",
"self": "Self",
"sh": "Shell",
"shaderlab": "ShaderLab",
"shell": "Shell",
"shell-script": "Shell",
"shellsession": "ShellSession",
"shen": "Shen",
"slash": "Slash",
"slim": "Slim",
"smali": "Smali",
"smalltalk": "Smalltalk",
"smarty": "Smarty",
"sml": "Standard ML",
"smt": "SMT",
"sourcemod": "SourcePawn",
"sourcepawn": "SourcePawn",
"sparql": "SPARQL",
"specfile": "RPM Spec",
"spline_font_database": "Spline Font Database",
"splus": "R",
"sqf": "SQF",
"sql": "SQL",
"sqlpl": "SQLPL",
"squeak": "Smalltalk",
"squirrel": "Squirrel",
"srecode_template": "SRecode Template",
"stan": "Stan",
"standard_ml": "Standard ML",
"stata": "Stata",
"ston": "STON",
"stylus": "Stylus",
"sublime_text_config": "Sublime Text Config",
"subrip_text": "SubRip Text",
"supercollider": "SuperCollider",
"svg": "SVG",
"swift": "Swift",
"systemverilog": "SystemVerilog",
"tcl": "Tcl",
"tcsh": "Tcsh",
"tea": "Tea",
"terra": "Terra",
"tex": "TeX",
"text": "Text",
"textile": "Textile",
"thrift": "Thrift",
"ti_program": "TI Program",
"tl": "Type Language",
"tla": "TLA",
"toml": "TOML",
"ts": "TypeScript",
"turing": "Turing",
"turtle": "Turtle",
"twig": "Twig",
"txl": "TXL",
"type_language": "Type Language",
"typescript": "TypeScript",
"udiff": "Diff",
"unified_parallel_c": "Unified Parallel C",
"unity3d_asset": "Unity3D Asset",
"unix_assembly": "Unix Assembly",
"uno": "Uno",
"unrealscript": "UnrealScript",
"ur": "UrWeb",
"ur/web": "UrWeb",
"urweb": "UrWeb",
"vala": "Vala",
"vb.net": "Visual Basic",
"vbnet": "Visual Basic",
"vcl": "VCL",
"verilog": "Verilog",
"vhdl": "VHDL",
"vim": "Vim script",
"vim_script": "Vim script",
"viml": "Vim script",
"visual_basic": "Visual Basic",
"volt": "Volt",
"vue": "Vue",
"wavefront_material": "Wavefront Material",
"wavefront_object": "Wavefront Object",
"web_ontology_language": "Web Ontology Language",
"webidl": "WebIDL",
"winbatch": "Batchfile",
"wisp": "wisp",
"world_of_warcraft_addon_data": "World of Warcraft Addon Data",
"wsdl": "XML",
"x10": "X10",
"xbase": "xBase",
"xc": "XC",
"xcompose": "XCompose",
"xhtml": "HTML",
"xml": "XML",
"xml+genshi": "Genshi",
"xml+kid": "Genshi",
"xojo": "Xojo",
"xpages": "XPages",
"xproc": "XProc",
"xquery": "XQuery",
"xs": "XS",
"xsd": "XML",
"xsl": "XSLT",
"xslt": "XSLT",
"xten": "X10",
"xtend": "Xtend",
"yacc": "Yacc",
"yaml": "YAML",
"yang": "YANG",
"yml": "YAML",
"zephir": "Zephir",
"zimpl": "Zimpl",
"zsh": "Shell",
}

51
classifier.go
View File

@ -6,38 +6,11 @@ import (
"gopkg.in/src-d/simple-linguist.v1/internal/tokenizer" "gopkg.in/src-d/simple-linguist.v1/internal/tokenizer"
) )
// GetLanguageByClassifier takes in a content and a list of candidates, and apply the classifier's Classify method to
// get the most probably language. If classifier is null then DefaultClassfier will be used.
func GetLanguageByClassifier(content []byte, candidates []string, classifier Classifier) string {
if classifier == nil {
classifier = DefaultClassifier
}
scores := classifier.Classify(content, candidates)
if len(scores) == 0 {
return OtherLanguage
}
return getLangugeHigherScore(scores)
}
func getLangugeHigherScore(scores map[string]float64) string {
var language string
higher := -math.MaxFloat64
for lang, score := range scores {
if higher < score {
language = lang
higher = score
}
}
return language
}
// Classifier is the interface that contains the method Classify which is in charge to assign scores to the possibles candidates. // Classifier is the interface that contains the method Classify which is in charge to assign scores to the possibles candidates.
// The scores must order the candidates so as the highest score be the most probably language of the content. // The scores must order the candidates so as the highest score be the most probably language of the content. The candidates is
// a map which can be used to assign weights to languages dynamically.
type Classifier interface { type Classifier interface {
Classify(content []byte, candidates []string) map[string]float64 Classify(content []byte, candidates map[string]float64) map[string]float64
} }
type classifier struct { type classifier struct {
@ -46,36 +19,36 @@ type classifier struct {
tokensTotal float64 tokensTotal float64
} }
func (c *classifier) Classify(content []byte, candidates []string) map[string]float64 { func (c *classifier) Classify(content []byte, candidates map[string]float64) map[string]float64 {
if len(content) == 0 { if len(content) == 0 {
return nil return nil
} }
var languages []string var languages map[string]float64
if len(candidates) == 0 { if len(candidates) == 0 {
languages = c.knownLangs() languages = c.knownLangs()
} else { } else {
languages = make([]string, 0, len(candidates)) languages = make(map[string]float64, len(candidates))
for _, candidate := range candidates { for candidate, weight := range candidates {
if lang, ok := GetLanguageByAlias(candidate); ok { if lang, ok := GetLanguageByAlias(candidate); ok {
languages = append(languages, lang) languages[lang] = weight
} }
} }
} }
tokens := tokenizer.Tokenize(content) tokens := tokenizer.Tokenize(content)
scores := make(map[string]float64, len(languages)) scores := make(map[string]float64, len(languages))
for _, language := range languages { for language := range languages {
scores[language] = c.tokensLogProbability(tokens, language) + c.languagesLogProbabilities[language] scores[language] = c.tokensLogProbability(tokens, language) + c.languagesLogProbabilities[language]
} }
return scores return scores
} }
func (c *classifier) knownLangs() []string { func (c *classifier) knownLangs() map[string]float64 {
langs := make([]string, 0, len(c.languagesLogProbabilities)) langs := make(map[string]float64, len(c.languagesLogProbabilities))
for lang := range c.languagesLogProbabilities { for lang := range c.languagesLogProbabilities {
langs = append(langs, lang) langs[lang]++
} }
return langs return langs

32
classifier_test.go
View File

@ -1,32 +0,0 @@
package slinguist
import (
"io/ioutil"
"path/filepath"
. "gopkg.in/check.v1"
)
func (s *TSuite) TestGetLanguageByClassifier(c *C) {
const samples = `.linguist/samples/`
test := []struct {
filename string
candidates []string
expectedLang string
}{
{filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"python", "ruby", "c", "c++"}, expectedLang: "C"},
{filename: filepath.Join(samples, "C/blob.c"), candidates: nil, expectedLang: "C"},
{filename: filepath.Join(samples, "C/main.c"), candidates: nil, expectedLang: "C"},
{filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"python", "ruby", "c++"}, expectedLang: "C++"},
{filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"ruby"}, expectedLang: "Ruby"},
{filename: filepath.Join(samples, "Python/django-models-base.py"), candidates: []string{"python", "ruby", "c", "c++"}, expectedLang: "Python"},
{filename: filepath.Join(samples, "Python/django-models-base.py"), candidates: nil, expectedLang: "Python"},
}
for _, test := range test {
content, err := ioutil.ReadFile(test.filename)
c.Assert(err, Equals, nil)
lang := GetLanguageByClassifier(content, test.candidates, nil)
c.Assert(lang, Equals, test.expectedLang)
}
}

34
cli/slinguist/main.go
View File

@ -21,7 +21,7 @@ func main() {
} }
errors := false errors := false
o := make(map[string][]string, 0) out := make(map[string][]string, 0)
err = filepath.Walk(root, func(path string, f os.FileInfo, err error) error { err = filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
if err != nil { if err != nil {
errors = true errors = true
@ -29,7 +29,23 @@ func main() {
return filepath.SkipDir return filepath.SkipDir
} }
if slinguist.IsVendor(f.Name()) || slinguist.IsDotFile(f.Name()) { relativePath, err := filepath.Rel(root, path)
if err != nil {
errors = true
log.Println(err)
return nil
}
if relativePath == "." {
return nil
}
if f.IsDir() {
relativePath = relativePath + "/"
}
if slinguist.IsVendor(relativePath) || slinguist.IsDotFile(relativePath) ||
slinguist.IsDocumentation(relativePath) || slinguist.IsConfiguration(relativePath) {
if f.IsDir() { if f.IsDir() {
return filepath.SkipDir return filepath.SkipDir
} }
@ -48,16 +64,12 @@ func main() {
return nil return nil
} }
l := slinguist.GetLanguage(filepath.Base(path), content) language := slinguist.GetLanguage(filepath.Base(path), content)
if language == slinguist.OtherLanguage {
r, err := filepath.Rel(root, path)
if err != nil {
errors = true
log.Println(err)
return nil return nil
} }
o[l] = append(o[l], r) out[language] = append(out[language], relativePath)
return nil return nil
}) })
@ -65,8 +77,8 @@ func main() {
log.Fatal(err) log.Fatal(err)
} }
js, _ := json.MarshalIndent(o, "", " ") data, _ := json.MarshalIndent(out, "", " ")
fmt.Printf("%s\n", js) fmt.Printf("%s\n", data)
if errors { if errors {
os.Exit(2) os.Exit(2)

210
common.go
View File

@ -1,71 +1,187 @@
package slinguist package slinguist
const OtherLanguage = "Other" import (
"math"
var ( "path/filepath"
ExtensionsByLanguage map[string][]string "strings"
ignoredExtensions = map[string]bool{
".asc": true, ".cgi": true, ".fcgi": true, ".gml": true, ".fx": true,
".vhost": true,
}
auxiliaryLanguages = map[string]bool{
"Other": true, "XML": true, "YAML": true, "TOML": true, "INI": true,
"JSON": true, "TeX": true, "Public Key": true, "AsciiDoc": true,
"AGS Script": true, "VimL": true, "Diff": true, "CMake": true, "fish": true,
"Awk": true, "Graphviz (DOT)": true, "Markdown": true, "desktop": true,
"XSLT": true, "SQL": true, "RMarkdown": true, "IRC log": true,
"reStructuredText": true, "Twig": true, "CSS": true, "Batchfile": true,
"Text": true, "HTML+ERB": true, "HTML": true, "Gettext Catalog": true,
"Smarty": true, "Raw token data": true,
}
) )
func init() { // OtherLanguage is used as a zero value when a function can not return a specific language.
for l, _ := range ignoredExtensions { const OtherLanguage = "Other"
languagesByExtension[l] = []string{OtherLanguage}
// Strategy type fix the signature for the functions that can be used as a strategy.
type Strategy func(filename string, content []byte) (languages []string)
var strategies = []Strategy{
GetLanguagesByModeline,
GetLanguagesByFilename,
GetLanguagesByShebang,
GetLanguagesByExtension,
GetLanguagesByContent,
} }
ExtensionsByLanguage = reverseStringListMap(languagesByExtension) // GetLanguage applies a sequence of strategies based on the given filename and content
} // to find out the most probably language to return.
// GetLanguageExtensions returns the different extensions being used by the
// language.
func GetLanguageExtensions(language string) []string {
return ExtensionsByLanguage[language]
}
// GetLanguage return the Language for a given filename and file content.
func GetLanguage(filename string, content []byte) string { func GetLanguage(filename string, content []byte) string {
if lang, safe := GetLanguageByModeline(content); safe { candidates := map[string]float64{}
for _, strategy := range strategies {
languages := strategy(filename, content)
if len(languages) == 1 {
return languages[0]
}
if len(languages) > 0 {
for _, language := range languages {
candidates[language]++
}
}
}
if len(candidates) == 0 {
return OtherLanguage
}
lang := GetLanguageByClassifier(content, candidates, nil)
return lang return lang
} }
if lang, safe := GetLanguageByFilename(filename); safe { // GetLanguageByModeline returns the language of the given content looking for the modeline,
return lang // and safe to indicate the sureness of returned language.
func GetLanguageByModeline(content []byte) (lang string, safe bool) {
return getLangAndSafe("", content, GetLanguagesByModeline)
} }
if lang, safe := GetLanguageByShebang(content); safe { // GetLanguageByFilename returns a language based on the given filename, and safe to indicate
return lang // the sureness of returned language.
func GetLanguageByFilename(filename string) (lang string, safe bool) {
return getLangAndSafe(filename, nil, GetLanguagesByFilename)
} }
if lang, safe := GetLanguageByExtension(filename); safe { // GetLanguagesByFilename returns a slice of possible languages for the given filename, content will be ignored.
return lang // It accomplish the signature to be a Strategy type.
func GetLanguagesByFilename(filename string, content []byte) []string {
return languagesByFilename[filename]
} }
if lang, safe := GetLanguageByContent(filename, content); safe { // GetLanguageByShebang returns the language of the given content looking for the shebang line,
return lang // and safe to indicate the sureness of returned language.
func GetLanguageByShebang(content []byte) (lang string, safe bool) {
return getLangAndSafe("", content, GetLanguagesByShebang)
} }
lang := GetLanguageByClassifier(content, nil, nil) // GetLanguageByExtension returns a language based on the given filename, and safe to indicate
return lang // the sureness of returned language.
func GetLanguageByExtension(filename string) (lang string, safe bool) {
return getLangAndSafe(filename, nil, GetLanguagesByExtension)
} }
func reverseStringListMap(i map[string][]string) (o map[string][]string) { // GetLanguagesByExtension returns a slice of possible languages for the given filename, content will be ignored.
o = map[string][]string{} // It accomplish the signature to be a Strategy type.
for key, set := range i { func GetLanguagesByExtension(filename string, content []byte) []string {
for _, value := range set { ext := strings.ToLower(filepath.Ext(filename))
o[value] = append(o[value], key) return languagesByExtension[ext]
} }
// GetLanguageByContent returns a language based on the filename and heuristics applies to the content,
// and safe to indicate the sureness of returned language.
func GetLanguageByContent(filename string, content []byte) (lang string, safe bool) {
return getLangAndSafe(filename, content, GetLanguagesByContent)
}
// GetLanguagesByContent returns a slice of possible languages for the given content, filename will be ignored.
// It accomplish the signature to be a Strategy type.
func GetLanguagesByContent(filename string, content []byte) []string {
ext := strings.ToLower(filepath.Ext(filename))
fnMatcher, ok := contentMatchers[ext]
if !ok {
return nil
}
return fnMatcher(content)
}
func getLangAndSafe(filename string, content []byte, getLanguageByStrategy Strategy) (lang string, safe bool) {
languages := getLanguageByStrategy(filename, content)
if len(languages) == 0 {
lang = OtherLanguage
return
}
lang = languages[0]
safe = len(languages) == 1
return
}
// GetLanguageByClassifier takes in a content and a list of candidates, and apply the classifier's Classify method to
// get the most probably language. If classifier is null then DefaultClassfier will be used. If there aren't candidates
// OtherLanguage is returned.
func GetLanguageByClassifier(content []byte, candidates map[string]float64, classifier Classifier) string {
scores := GetLanguagesByClassifier(content, candidates, classifier)
if len(scores) == 0 {
return OtherLanguage
}
return getLangugeHigherScore(scores)
}
func getLangugeHigherScore(scores map[string]float64) string {
var language string
higher := -math.MaxFloat64
for lang, score := range scores {
if higher < score {
language = lang
higher = score
}
}
return language
}
// GetLanguagesByClassifier returns a map of possible languages as keys and a score as value based on content and candidates. The values can be ordered
// with the highest value as the most probably language. If classifier is null then DefaultClassfier will be used.
func GetLanguagesByClassifier(content []byte, candidates map[string]float64, classifier Classifier) map[string]float64 {
if classifier == nil {
classifier = DefaultClassifier
}
return classifier.Classify(content, candidates)
}
// GetLanguageExtensions returns the different extensions being used by the language.
func GetLanguageExtensions(language string) []string {
return extensionsByLanguage[language]
}
// Type represent language's type. Either data, programming, markup, prose, or unknown.
type Type int
// Type's values.
const (
Unknown Type = iota
Data
Programming
Markup
Prose
)
// GetLanguageType returns the given language's type.
func GetLanguageType(language string) (langType Type) {
langType, ok := languagesType[language]
if !ok {
langType = Unknown
}
return langType
}
// GetLanguageByAlias returns either the language related to the given alias and ok set to true
// or Otherlanguage and ok set to false if the alias is not recognized.
func GetLanguageByAlias(alias string) (lang string, ok bool) {
a := strings.Split(alias, `,`)[0]
a = strings.ToLower(a)
lang, ok = languagesByAlias[a]
if !ok {
lang = OtherLanguage
} }
return return

354
common_test.go
View File

@ -1,19 +1,351 @@
package slinguist package slinguist
import ( import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing" "testing"
"text/tabwriter"
. "gopkg.in/check.v1" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
) )
func Test(t *testing.T) { TestingT(t) } type SimpleLinguistTestSuite struct {
suite.Suite
type TSuite struct{} }
var _ = Suite(&TSuite{}) func TestSimpleLinguistTestSuite(t *testing.T) {
suite.Run(t, new(SimpleLinguistTestSuite))
func (s *TSuite) TestGetLanguage(c *C) { }
c.Assert(GetLanguage("foo.py", []byte{}), Equals, "Python")
c.Assert(GetLanguage("foo.m", []byte(":- module")), Equals, "Mercury") func (s *SimpleLinguistTestSuite) TestGetLanguage() {
c.Assert(GetLanguage("foo.m", []byte{}), Equals, "Other") tests := []struct {
name string
filename string
content []byte
expected string
}{
{name: "TestGetLanguage_1", filename: "foo.py", content: []byte{}, expected: "Python"},
{name: "TestGetLanguage_2", filename: "foo.m", content: []byte(":- module"), expected: "Mercury"},
{name: "TestGetLanguage_3", filename: "foo.m", content: nil, expected: OtherLanguage},
}
for _, test := range tests {
language := GetLanguage(test.filename, test.content)
assert.Equal(s.T(), test.expected, language, fmt.Sprintf("%v: %v, expected: %v", test.name, language, test.expected))
}
}
func (s *SimpleLinguistTestSuite) TestGetLanguageByModelineLinguist() {
const (
modelinesDir = ".linguist/test/fixtures/Data/Modelines"
samplesDir = ".linguist/samples"
)
tests := []struct {
name string
filename string
expectedLang string
expectedSafe bool
}{
// Emacs
{name: "TestGetLanguageByModelineLinguist_1", filename: filepath.Join(modelinesDir, "example_smalltalk.md"), expectedLang: "Smalltalk", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_2", filename: filepath.Join(modelinesDir, "fundamentalEmacs.c"), expectedLang: "Text", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_3", filename: filepath.Join(modelinesDir, "iamphp.inc"), expectedLang: "PHP", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_4", filename: filepath.Join(modelinesDir, "seeplusplusEmacs1"), expectedLang: "C++", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_5", filename: filepath.Join(modelinesDir, "seeplusplusEmacs2"), expectedLang: "C++", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_6", filename: filepath.Join(modelinesDir, "seeplusplusEmacs3"), expectedLang: "C++", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_7", filename: filepath.Join(modelinesDir, "seeplusplusEmacs4"), expectedLang: "C++", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_8", filename: filepath.Join(modelinesDir, "seeplusplusEmacs5"), expectedLang: "C++", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_9", filename: filepath.Join(modelinesDir, "seeplusplusEmacs6"), expectedLang: "C++", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_10", filename: filepath.Join(modelinesDir, "seeplusplusEmacs7"), expectedLang: "C++", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_11", filename: filepath.Join(modelinesDir, "seeplusplusEmacs9"), expectedLang: "C++", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_12", filename: filepath.Join(modelinesDir, "seeplusplusEmacs10"), expectedLang: "C++", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_13", filename: filepath.Join(modelinesDir, "seeplusplusEmacs11"), expectedLang: "C++", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_14", filename: filepath.Join(modelinesDir, "seeplusplusEmacs12"), expectedLang: "C++", expectedSafe: true},
// Vim
{name: "TestGetLanguageByModelineLinguist_15", filename: filepath.Join(modelinesDir, "seeplusplus"), expectedLang: "C++", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_16", filename: filepath.Join(modelinesDir, "iamjs.pl"), expectedLang: "JavaScript", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_17", filename: filepath.Join(modelinesDir, "iamjs2.pl"), expectedLang: "JavaScript", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_18", filename: filepath.Join(modelinesDir, "not_perl.pl"), expectedLang: "Prolog", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_19", filename: filepath.Join(modelinesDir, "ruby"), expectedLang: "Ruby", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_20", filename: filepath.Join(modelinesDir, "ruby2"), expectedLang: "Ruby", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_21", filename: filepath.Join(modelinesDir, "ruby3"), expectedLang: "Ruby", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_22", filename: filepath.Join(modelinesDir, "ruby4"), expectedLang: "Ruby", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_23", filename: filepath.Join(modelinesDir, "ruby5"), expectedLang: "Ruby", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_24", filename: filepath.Join(modelinesDir, "ruby6"), expectedLang: "Ruby", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_25", filename: filepath.Join(modelinesDir, "ruby7"), expectedLang: "Ruby", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_26", filename: filepath.Join(modelinesDir, "ruby8"), expectedLang: "Ruby", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_27", filename: filepath.Join(modelinesDir, "ruby9"), expectedLang: "Ruby", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_28", filename: filepath.Join(modelinesDir, "ruby10"), expectedLang: "Ruby", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_29", filename: filepath.Join(modelinesDir, "ruby11"), expectedLang: "Ruby", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_30", filename: filepath.Join(modelinesDir, "ruby12"), expectedLang: "Ruby", expectedSafe: true},
{name: "TestGetLanguageByModelineLinguist_31", filename: filepath.Join(samplesDir, "C/main.c"), expectedLang: OtherLanguage, expectedSafe: false},
}
for _, test := range tests {
content, err := ioutil.ReadFile(test.filename)
assert.NoError(s.T(), err)
lang, safe := GetLanguageByModeline(content)
assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang))
assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe))
}
}
func (s *SimpleLinguistTestSuite) TestGetLanguageByModeline() {
const (
wrongVim = `# vim: set syntax=ruby ft =python filetype=perl :`
rightVim = `/* vim: set syntax=python ft =python filetype=python */`
noLangVim = `/* vim: set shiftwidth=4 softtabstop=0 cindent cinoptions={1s: */`
)
tests := []struct {
name string
content []byte
expectedLang string
expectedSafe bool
}{
{name: "TestGetLanguageByModeline_1", content: []byte(wrongVim), expectedLang: OtherLanguage, expectedSafe: false},
{name: "TestGetLanguageByModeline_2", content: []byte(rightVim), expectedLang: "Python", expectedSafe: true},
{name: "TestGetLanguageByModeline_3", content: []byte(noLangVim), expectedLang: OtherLanguage, expectedSafe: false},
}
for _, test := range tests {
lang, safe := GetLanguageByModeline(test.content)
assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang))
assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe))
}
}
func (s *SimpleLinguistTestSuite) TestGetLanguageByFilename() {
tests := []struct {
name string
filename string
expectedLang string
expectedSafe bool
}{
{name: "TestGetLanguageByFilename_1", filename: "unknown.interpreter", expectedLang: OtherLanguage, expectedSafe: false},
{name: "TestGetLanguageByFilename_2", filename: ".bashrc", expectedLang: "Shell", expectedSafe: true},
{name: "TestGetLanguageByFilename_3", filename: "Dockerfile", expectedLang: "Dockerfile", expectedSafe: true},
{name: "TestGetLanguageByFilename_4", filename: "Makefile.frag", expectedLang: "Makefile", expectedSafe: true},
{name: "TestGetLanguageByFilename_5", filename: "makefile", expectedLang: "Makefile", expectedSafe: true},
{name: "TestGetLanguageByFilename_6", filename: "Vagrantfile", expectedLang: "Ruby", expectedSafe: true},
{name: "TestGetLanguageByFilename_7", filename: "_vimrc", expectedLang: "Vim script", expectedSafe: true},
{name: "TestGetLanguageByFilename_8", filename: "pom.xml", expectedLang: "Maven POM", expectedSafe: true},
}
for _, test := range tests {
lang, safe := GetLanguageByFilename(test.filename)
assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang))
assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe))
}
}
func (s *SimpleLinguistTestSuite) TestGetLanguageByShebang() {
const (
multilineExecHack = `#!/bin/sh
# Next line is comment in Tcl, but not in sh... \
exec tclsh "$0" ${1+"$@"}`
multilineNoExecHack = `#!/bin/sh
#<<<#
echo "A shell script in a zkl program ($0)"
echo "Now run zkl <this file> with Hello World as args"
zkl $0 Hello World!
exit
#<<<#
println("The shell script says ",vm.arglist.concat(" "));`
)
tests := []struct {
name string
content []byte
expectedLang string
expectedSafe bool
}{
{name: "TestGetLanguageByShebang_1", content: []byte(`#!/unknown/interpreter`), expectedLang: OtherLanguage, expectedSafe: false},
{name: "TestGetLanguageByShebang_2", content: []byte(`no shebang`), expectedLang: OtherLanguage, expectedSafe: false},
{name: "TestGetLanguageByShebang_3", content: []byte(`#!/usr/bin/env`), expectedLang: OtherLanguage, expectedSafe: false},
{name: "TestGetLanguageByShebang_4", content: []byte(`#!/usr/bin/python -tt`), expectedLang: "Python", expectedSafe: true},
{name: "TestGetLanguageByShebang_5", content: []byte(`#!/usr/bin/env python2.6`), expectedLang: "Python", expectedSafe: true},
{name: "TestGetLanguageByShebang_6", content: []byte(`#!/usr/bin/env perl`), expectedLang: "Perl", expectedSafe: true},
{name: "TestGetLanguageByShebang_7", content: []byte(`#! /bin/sh`), expectedLang: "Shell", expectedSafe: true},
{name: "TestGetLanguageByShebang_8", content: []byte(`#!bash`), expectedLang: "Shell", expectedSafe: true},
{name: "TestGetLanguageByShebang_9", content: []byte(multilineExecHack), expectedLang: "Tcl", expectedSafe: true},
{name: "TestGetLanguageByShebang_10", content: []byte(multilineNoExecHack), expectedLang: "Shell", expectedSafe: true},
}
for _, test := range tests {
lang, safe := GetLanguageByShebang(test.content)
assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang))
assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe))
}
}
func (s *SimpleLinguistTestSuite) TestGetLanguageByExtension() {
tests := []struct {
name string
filename string
expectedLang string
expectedSafe bool
}{
{name: "TestGetLanguageByExtension_1", filename: "foo.foo", expectedLang: OtherLanguage, expectedSafe: false},
{name: "TestGetLanguageByExtension_2", filename: "foo.go", expectedLang: "Go", expectedSafe: true},
{name: "TestGetLanguageByExtension_3", filename: "foo.go.php", expectedLang: "Hack", expectedSafe: false},
}
for _, test := range tests {
lang, safe := GetLanguageByExtension(test.filename)
assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang))
assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe))
}
}
func (s *SimpleLinguistTestSuite) TestGetLanguageByContentLinguistCorpus() {
var total, failed, ok, other, unsafe int
w := new(tabwriter.Writer)
w.Init(os.Stdout, 0, 8, 0, '\t', 0)
filepath.Walk(".linguist/samples", func(path string, f os.FileInfo, err error) error {
if f.IsDir() {
if f.Name() == "filenames" {
return filepath.SkipDir
}
return nil
}
expected := filepath.Base(filepath.Dir(path))
filename := filepath.Base(path)
extension := filepath.Ext(path)
content, _ := ioutil.ReadFile(path)
if extension == "" {
return nil
}
total++
obtained, safe := GetLanguageByContent(filename, content)
if obtained == OtherLanguage {
other++
}
var status string
if expected == obtained {
status = "ok"
ok++
} else {
status = "failed"
failed++
if !safe {
unsafe++
}
}
fmt.Fprintf(w, "%s\t%s\t%s\t%v\t%s\n", filename, expected, obtained, safe, status)
return nil
})
fmt.Fprintln(w)
w.Flush()
fmt.Printf("total files: %d, ok: %d, failed: %d, unsafe: %d, other: %d\n", total, ok, failed, unsafe, other)
}
func (s *SimpleLinguistTestSuite) TestGetLanguageByClassifier() {
const samples = `.linguist/samples/`
test := []struct {
name string
filename string
candidates map[string]float64
expected string
}{
{name: "TestGetLanguageByClassifier_1", filename: filepath.Join(samples, "C/blob.c"), candidates: map[string]float64{"python": 1.00, "ruby": 1.00, "c": 1.00, "c++": 1.00}, expected: "C"},
{name: "TestGetLanguageByClassifier_2", filename: filepath.Join(samples, "C/blob.c"), candidates: nil, expected: "C"},
{name: "TestGetLanguageByClassifier_3", filename: filepath.Join(samples, "C/main.c"), candidates: nil, expected: "C"},
{name: "TestGetLanguageByClassifier_4", filename: filepath.Join(samples, "C/blob.c"), candidates: map[string]float64{"python": 1.00, "ruby": 1.00, "c++": 1.00}, expected: "C++"},
{name: "TestGetLanguageByClassifier_5", filename: filepath.Join(samples, "C/blob.c"), candidates: map[string]float64{"ruby": 1.00}, expected: "Ruby"},
{name: "TestGetLanguageByClassifier_6", filename: filepath.Join(samples, "Python/django-models-base.py"), candidates: map[string]float64{"python": 1.00, "ruby": 1.00, "c": 1.00, "c++": 1.00}, expected: "Python"},
{name: "TestGetLanguageByClassifier_7", filename: filepath.Join(samples, "Python/django-models-base.py"), candidates: nil, expected: "Python"},
}
for _, test := range test {
content, err := ioutil.ReadFile(test.filename)
assert.NoError(s.T(), err)
lang := GetLanguageByClassifier(content, test.candidates, nil)
assert.Equal(s.T(), test.expected, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expected))
}
}
func (s *SimpleLinguistTestSuite) TestGetLanguageExtensions() {
tests := []struct {
name string
language string
expected []string
}{
{name: "TestGetLanguageExtensions_1", language: "foo", expected: nil},
{name: "TestGetLanguageExtensions_2", language: "COBOL", expected: []string{".cob", ".cbl", ".ccp", ".cobol", ".cpy"}},
{name: "TestGetLanguageExtensions_3", language: "Maven POM", expected: nil},
}
for _, test := range tests {
extensions := GetLanguageExtensions(test.language)
assert.EqualValues(s.T(), test.expected, extensions, fmt.Sprintf("%v: extensions = %v, expected: %v", test.name, extensions, test.expected))
}
}
func (s *SimpleLinguistTestSuite) TestGetLanguageType() {
tests := []struct {
name string
language string
expected Type
}{
{name: "TestGetLanguageType_1", language: "BestLanguageEver", expected: Unknown},
{name: "TestGetLanguageType_2", language: "JSON", expected: Data},
{name: "TestGetLanguageType_3", language: "COLLADA", expected: Data},
{name: "TestGetLanguageType_4", language: "Go", expected: Programming},
{name: "TestGetLanguageType_5", language: "Brainfuck", expected: Programming},
{name: "TestGetLanguageType_6", language: "HTML", expected: Markup},
{name: "TestGetLanguageType_7", language: "Sass", expected: Markup},
{name: "TestGetLanguageType_8", language: "AsciiDoc", expected: Prose},
{name: "TestGetLanguageType_9", language: "Textile", expected: Prose},
}
for _, test := range tests {
langType := GetLanguageType(test.language)
assert.Equal(s.T(), test.expected, langType, fmt.Sprintf("%v: langType = %v, expected: %v", test.name, langType, test.expected))
}
}
func (s *SimpleLinguistTestSuite) TestGetLanguageByAlias() {
tests := []struct {
name string
alias string
expectedLang string
expectedOk bool
}{
{name: "TestGetLanguageByAlias_1", alias: "BestLanguageEver", expectedLang: OtherLanguage, expectedOk: false},
{name: "TestGetLanguageByAlias_2", alias: "aspx-vb", expectedLang: "ASP", expectedOk: true},
{name: "TestGetLanguageByAlias_3", alias: "C++", expectedLang: "C++", expectedOk: true},
{name: "TestGetLanguageByAlias_4", alias: "c++", expectedLang: "C++", expectedOk: true},
{name: "TestGetLanguageByAlias_5", alias: "objc", expectedLang: "Objective-C", expectedOk: true},
{name: "TestGetLanguageByAlias_6", alias: "golang", expectedLang: "Go", expectedOk: true},
{name: "TestGetLanguageByAlias_7", alias: "GOLANG", expectedLang: "Go", expectedOk: true},
{name: "TestGetLanguageByAlias_8", alias: "bsdmake", expectedLang: "Makefile", expectedOk: true},
{name: "TestGetLanguageByAlias_9", alias: "xhTmL", expectedLang: "HTML", expectedOk: true},
{name: "TestGetLanguageByAlias_10", alias: "python", expectedLang: "Python", expectedOk: true},
}
for _, test := range tests {
lang, ok := GetLanguageByAlias(test.alias)
assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang))
assert.Equal(s.T(), test.expectedOk, ok, fmt.Sprintf("%v: ok = %v, expected: %v", test.name, ok, test.expectedOk))
}
} }

405
content.go
View File

@ -2,460 +2,447 @@ package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND // THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: 60f864a138650dd17fafc94814be9ee2d3aaef8c // Extracted from github/linguist commit: b6460f8ed6b249281ada099ca28bd8f1230b8892
import ( import (
"path/filepath"
"regexp" "regexp"
"strings"
) )
func GetLanguageByContent(filename string, content []byte) (lang string, safe bool) { type languageMatcher func([]byte) []string
ext := strings.ToLower(filepath.Ext(filename))
if fnMatcher, ok := matchers[ext]; ok {
lang, safe = fnMatcher(content)
} else {
lang = OtherLanguage
}
return var contentMatchers = map[string]languageMatcher{
} ".asc": func(i []byte) []string {
type languageMatcher func([]byte) (string, bool)
var matchers = map[string]languageMatcher{
".asc": func(i []byte) (string, bool) {
if asc_PublicKey_Matcher_0.Match(i) { if asc_PublicKey_Matcher_0.Match(i) {
return "Public Key", true return []string{"Public Key"}
} else if asc_AsciiDoc_Matcher_0.Match(i) { } else if asc_AsciiDoc_Matcher_0.Match(i) {
return "AsciiDoc", true return []string{"AsciiDoc"}
} else if asc_AGSScript_Matcher_0.Match(i) { } else if asc_AGSScript_Matcher_0.Match(i) {
return "AGS Script", true return []string{"AGS Script"}
} }
return OtherLanguage, false return nil
}, },
".bb": func(i []byte) (string, bool) { ".bb": func(i []byte) []string {
if bb_BlitzBasic_Matcher_0.Match(i) || bb_BlitzBasic_Matcher_1.Match(i) { if bb_BlitzBasic_Matcher_0.Match(i) || bb_BlitzBasic_Matcher_1.Match(i) {
return "BlitzBasic", true return []string{"BlitzBasic"}
} else if bb_BitBake_Matcher_0.Match(i) { } else if bb_BitBake_Matcher_0.Match(i) {
return "BitBake", true return []string{"BitBake"}
} }
return OtherLanguage, false return nil
}, },
".builds": func(i []byte) (string, bool) { ".builds": func(i []byte) []string {
if builds_XML_Matcher_0.Match(i) { if builds_XML_Matcher_0.Match(i) {
return "XML", true return []string{"XML"}
} }
return "Text", true return []string{"Text"}
}, },
".ch": func(i []byte) (string, bool) { ".ch": func(i []byte) []string {
if ch_xBase_Matcher_0.Match(i) { if ch_xBase_Matcher_0.Match(i) {
return "xBase", true return []string{"xBase"}
} }
return OtherLanguage, false return nil
}, },
".cl": func(i []byte) (string, bool) { ".cl": func(i []byte) []string {
if cl_CommonLisp_Matcher_0.Match(i) { if cl_CommonLisp_Matcher_0.Match(i) {
return "Common Lisp", true return []string{"Common Lisp"}
} else if cl_Cool_Matcher_0.Match(i) { } else if cl_Cool_Matcher_0.Match(i) {
return "Cool", true return []string{"Cool"}
} else if cl_OpenCL_Matcher_0.Match(i) { } else if cl_OpenCL_Matcher_0.Match(i) {
return "OpenCL", true return []string{"OpenCL"}
} }
return OtherLanguage, false return nil
}, },
".cls": func(i []byte) (string, bool) { ".cls": func(i []byte) []string {
if cls_TeX_Matcher_0.Match(i) { if cls_TeX_Matcher_0.Match(i) {
return "TeX", true return []string{"TeX"}
} }
return OtherLanguage, false return nil
}, },
".cs": func(i []byte) (string, bool) { ".cs": func(i []byte) []string {
if cs_Smalltalk_Matcher_0.Match(i) { if cs_Smalltalk_Matcher_0.Match(i) {
return "Smalltalk", true return []string{"Smalltalk"}
} else if cs_CSharp_Matcher_0.Match(i) || cs_CSharp_Matcher_1.Match(i) { } else if cs_CSharp_Matcher_0.Match(i) || cs_CSharp_Matcher_1.Match(i) {
return "C#", true return []string{"C#"}
} }
return OtherLanguage, false return nil
}, },
".d": func(i []byte) (string, bool) { ".d": func(i []byte) []string {
if d_D_Matcher_0.Match(i) { if d_D_Matcher_0.Match(i) {
return "D", true return []string{"D"}
} else if d_DTrace_Matcher_0.Match(i) { } else if d_DTrace_Matcher_0.Match(i) {
return "DTrace", true return []string{"DTrace"}
} else if d_Makefile_Matcher_0.Match(i) { } else if d_Makefile_Matcher_0.Match(i) {
return "Makefile", true return []string{"Makefile"}
} }
return OtherLanguage, false return nil
}, },
".ecl": func(i []byte) (string, bool) { ".ecl": func(i []byte) []string {
if ecl_ECLiPSe_Matcher_0.Match(i) { if ecl_ECLiPSe_Matcher_0.Match(i) {
return "ECLiPSe", true return []string{"ECLiPSe"}
} else if ecl_ECL_Matcher_0.Match(i) { } else if ecl_ECL_Matcher_0.Match(i) {
return "ECL", true return []string{"ECL"}
} }
return OtherLanguage, false return nil
}, },
".es": func(i []byte) (string, bool) { ".es": func(i []byte) []string {
if es_Erlang_Matcher_0.Match(i) { if es_Erlang_Matcher_0.Match(i) {
return "Erlang", true return []string{"Erlang"}
} }
return OtherLanguage, false return nil
}, },
".f": func(i []byte) (string, bool) { ".f": func(i []byte) []string {
if f_Forth_Matcher_0.Match(i) { if f_Forth_Matcher_0.Match(i) {
return "Forth", true return []string{"Forth"}
} else if f_FilebenchWML_Matcher_0.Match(i) { } else if f_FilebenchWML_Matcher_0.Match(i) {
return "Filebench WML", true return []string{"Filebench WML"}
} else if f_Fortran_Matcher_0.Match(i) { } else if f_Fortran_Matcher_0.Match(i) {
return "Fortran", true return []string{"Fortran"}
} }
return OtherLanguage, false return nil
}, },
".for": func(i []byte) (string, bool) { ".for": func(i []byte) []string {
if for_Forth_Matcher_0.Match(i) { if for_Forth_Matcher_0.Match(i) {
return "Forth", true return []string{"Forth"}
} else if for_Fortran_Matcher_0.Match(i) { } else if for_Fortran_Matcher_0.Match(i) {
return "Fortran", true return []string{"Fortran"}
} }
return OtherLanguage, false return nil
}, },
".fr": func(i []byte) (string, bool) { ".fr": func(i []byte) []string {
if fr_Forth_Matcher_0.Match(i) { if fr_Forth_Matcher_0.Match(i) {
return "Forth", true return []string{"Forth"}
} else if fr_Frege_Matcher_0.Match(i) { } else if fr_Frege_Matcher_0.Match(i) {
return "Frege", true return []string{"Frege"}
} }
return "Text", true return []string{"Text"}
}, },
".fs": func(i []byte) (string, bool) { ".fs": func(i []byte) []string {
if fs_Forth_Matcher_0.Match(i) { if fs_Forth_Matcher_0.Match(i) {
return "Forth", true return []string{"Forth"}
} else if fs_FSharp_Matcher_0.Match(i) { } else if fs_FSharp_Matcher_0.Match(i) {
return "F#", true return []string{"F#"}
} else if fs_GLSL_Matcher_0.Match(i) { } else if fs_GLSL_Matcher_0.Match(i) {
return "GLSL", true return []string{"GLSL"}
} else if fs_Filterscript_Matcher_0.Match(i) { } else if fs_Filterscript_Matcher_0.Match(i) {
return "Filterscript", true return []string{"Filterscript"}
} }
return OtherLanguage, false return nil
}, },
".gs": func(i []byte) (string, bool) { ".gs": func(i []byte) []string {
if gs_Gosu_Matcher_0.Match(i) { if gs_Gosu_Matcher_0.Match(i) {
return "Gosu", true return []string{"Gosu"}
} }
return OtherLanguage, false return nil
}, },
".h": func(i []byte) (string, bool) { ".h": func(i []byte) []string {
if h_ObjectiveDashC_Matcher_0.Match(i) { if h_ObjectiveDashC_Matcher_0.Match(i) {
return "Objective-C", true return []string{"Objective-C"}
} else if h_CPlusPlus_Matcher_0.Match(i) || h_CPlusPlus_Matcher_1.Match(i) || h_CPlusPlus_Matcher_2.Match(i) || h_CPlusPlus_Matcher_3.Match(i) || h_CPlusPlus_Matcher_4.Match(i) || h_CPlusPlus_Matcher_5.Match(i) || h_CPlusPlus_Matcher_6.Match(i) { } else if h_CPlusPlus_Matcher_0.Match(i) || h_CPlusPlus_Matcher_1.Match(i) || h_CPlusPlus_Matcher_2.Match(i) || h_CPlusPlus_Matcher_3.Match(i) || h_CPlusPlus_Matcher_4.Match(i) || h_CPlusPlus_Matcher_5.Match(i) || h_CPlusPlus_Matcher_6.Match(i) {
return "C++", true return []string{"C++"}
} }
return OtherLanguage, false return nil
}, },
".inc": func(i []byte) (string, bool) { ".inc": func(i []byte) []string {
if inc_PHP_Matcher_0.Match(i) { if inc_PHP_Matcher_0.Match(i) {
return "PHP", true return []string{"PHP"}
} else if inc_POVDashRaySDL_Matcher_0.Match(i) { } else if inc_POVDashRaySDL_Matcher_0.Match(i) {
return "POV-Ray SDL", true return []string{"POV-Ray SDL"}
} }
return OtherLanguage, false return nil
}, },
".l": func(i []byte) (string, bool) { ".l": func(i []byte) []string {
if l_CommonLisp_Matcher_0.Match(i) { if l_CommonLisp_Matcher_0.Match(i) {
return "Common Lisp", true return []string{"Common Lisp"}
} else if l_Lex_Matcher_0.Match(i) { } else if l_Lex_Matcher_0.Match(i) {
return "Lex", true return []string{"Lex"}
} else if l_Roff_Matcher_0.Match(i) { } else if l_Roff_Matcher_0.Match(i) {
return "Roff", true return []string{"Roff"}
} else if l_PicoLisp_Matcher_0.Match(i) { } else if l_PicoLisp_Matcher_0.Match(i) {
return "PicoLisp", true return []string{"PicoLisp"}
} }
return OtherLanguage, false return nil
}, },
".ls": func(i []byte) (string, bool) { ".ls": func(i []byte) []string {
if ls_LoomScript_Matcher_0.Match(i) { if ls_LoomScript_Matcher_0.Match(i) {
return "LoomScript", true return []string{"LoomScript"}
} }
return "LiveScript", true return []string{"LiveScript"}
}, },
".lsp": func(i []byte) (string, bool) { ".lsp": func(i []byte) []string {
if lsp_CommonLisp_Matcher_0.Match(i) { if lsp_CommonLisp_Matcher_0.Match(i) {
return "Common Lisp", true return []string{"Common Lisp"}
} else if lsp_NewLisp_Matcher_0.Match(i) { } else if lsp_NewLisp_Matcher_0.Match(i) {
return "NewLisp", true return []string{"NewLisp"}
} }
return OtherLanguage, false return nil
}, },
".lisp": func(i []byte) (string, bool) { ".lisp": func(i []byte) []string {
if lisp_CommonLisp_Matcher_0.Match(i) { if lisp_CommonLisp_Matcher_0.Match(i) {
return "Common Lisp", true return []string{"Common Lisp"}
} else if lisp_NewLisp_Matcher_0.Match(i) { } else if lisp_NewLisp_Matcher_0.Match(i) {
return "NewLisp", true return []string{"NewLisp"}
} }
return OtherLanguage, false return nil
}, },
".m": func(i []byte) (string, bool) { ".m": func(i []byte) []string {
if m_ObjectiveDashC_Matcher_0.Match(i) { if m_ObjectiveDashC_Matcher_0.Match(i) {
return "Objective-C", true return []string{"Objective-C"}
} else if m_Mercury_Matcher_0.Match(i) { } else if m_Mercury_Matcher_0.Match(i) {
return "Mercury", true return []string{"Mercury"}
} else if m_MUF_Matcher_0.Match(i) { } else if m_MUF_Matcher_0.Match(i) {
return "MUF", true return []string{"MUF"}
} else if m_M_Matcher_0.Match(i) { } else if m_M_Matcher_0.Match(i) {
return "M", true return []string{"M"}
} else if m_Mathematica_Matcher_0.Match(i) { } else if m_Mathematica_Matcher_0.Match(i) {
return "Mathematica", true return []string{"Mathematica"}
} else if m_Matlab_Matcher_0.Match(i) { } else if m_Matlab_Matcher_0.Match(i) {
return "Matlab", true return []string{"Matlab"}
} else if m_Limbo_Matcher_0.Match(i) { } else if m_Limbo_Matcher_0.Match(i) {
return "Limbo", true return []string{"Limbo"}
} }
return OtherLanguage, false return nil
}, },
".md": func(i []byte) (string, bool) { ".md": func(i []byte) []string {
if md_Markdown_Matcher_0.Match(i) || md_Markdown_Matcher_1.Match(i) { if md_Markdown_Matcher_0.Match(i) || md_Markdown_Matcher_1.Match(i) {
return "Markdown", true return []string{"Markdown"}
} else if md_GCCMachineDescription_Matcher_0.Match(i) { } else if md_GCCMachineDescription_Matcher_0.Match(i) {
return "GCC Machine Description", true return []string{"GCC Machine Description"}
} }
return "Markdown", true return []string{"Markdown"}
}, },
".ml": func(i []byte) (string, bool) { ".ml": func(i []byte) []string {
if ml_OCaml_Matcher_0.Match(i) { if ml_OCaml_Matcher_0.Match(i) {
return "OCaml", true return []string{"OCaml"}
} else if ml_StandardML_Matcher_0.Match(i) { } else if ml_StandardML_Matcher_0.Match(i) {
return "Standard ML", true return []string{"Standard ML"}
} }
return OtherLanguage, false return nil
}, },
".mod": func(i []byte) (string, bool) { ".mod": func(i []byte) []string {
if mod_XML_Matcher_0.Match(i) { if mod_XML_Matcher_0.Match(i) {
return "XML", true return []string{"XML"}
} else if mod_ModulaDash2_Matcher_0.Match(i) || mod_ModulaDash2_Matcher_1.Match(i) { } else if mod_ModulaDash2_Matcher_0.Match(i) || mod_ModulaDash2_Matcher_1.Match(i) {
return "Modula-2", true return []string{"Modula-2"}
} }
return "Linux Kernel Module", false return []string{"Linux Kernel Module", "AMPL"}
}, },
".ms": func(i []byte) (string, bool) { ".ms": func(i []byte) []string {
if ms_Roff_Matcher_0.Match(i) { if ms_Roff_Matcher_0.Match(i) {
return "Roff", true return []string{"Roff"}
} }
return "MAXScript", true return []string{"MAXScript"}
}, },
".n": func(i []byte) (string, bool) { ".n": func(i []byte) []string {
if n_Roff_Matcher_0.Match(i) { if n_Roff_Matcher_0.Match(i) {
return "Roff", true return []string{"Roff"}
} else if n_Nemerle_Matcher_0.Match(i) { } else if n_Nemerle_Matcher_0.Match(i) {
return "Nemerle", true return []string{"Nemerle"}
} }
return OtherLanguage, false return nil
}, },
".ncl": func(i []byte) (string, bool) { ".ncl": func(i []byte) []string {
if ncl_Text_Matcher_0.Match(i) { if ncl_Text_Matcher_0.Match(i) {
return "Text", true return []string{"Text"}
} }
return OtherLanguage, false return nil
}, },
".nl": func(i []byte) (string, bool) { ".nl": func(i []byte) []string {
if nl_NL_Matcher_0.Match(i) { if nl_NL_Matcher_0.Match(i) {
return "NL", true return []string{"NL"}
} }
return "NewLisp", true return []string{"NewLisp"}
}, },
".php": func(i []byte) (string, bool) { ".php": func(i []byte) []string {
if php_Hack_Matcher_0.Match(i) { if php_Hack_Matcher_0.Match(i) {
return "Hack", true return []string{"Hack"}
} else if php_PHP_Matcher_0.Match(i) { } else if php_PHP_Matcher_0.Match(i) {
return "PHP", true return []string{"PHP"}
} }
return OtherLanguage, false return nil
}, },
".pl": func(i []byte) (string, bool) { ".pl": func(i []byte) []string {
if pl_Prolog_Matcher_0.Match(i) { if pl_Prolog_Matcher_0.Match(i) {
return "Prolog", true return []string{"Prolog"}
} else if pl_Perl_Matcher_0.Match(i) { } else if pl_Perl_Matcher_0.Match(i) {
return "Perl", true return []string{"Perl"}
} else if pl_Perl6_Matcher_0.Match(i) { } else if pl_Perl6_Matcher_0.Match(i) {
return "Perl6", true return []string{"Perl6"}
} }
return OtherLanguage, false return nil
}, },
".pm": func(i []byte) (string, bool) { ".pm": func(i []byte) []string {
if pm_Perl6_Matcher_0.Match(i) { if pm_Perl6_Matcher_0.Match(i) {
return "Perl6", true return []string{"Perl6"}
} else if pm_Perl_Matcher_0.Match(i) { } else if pm_Perl_Matcher_0.Match(i) {
return "Perl", true return []string{"Perl"}
} }
return OtherLanguage, false return nil
}, },
".pod": func(i []byte) (string, bool) { ".pod": func(i []byte) []string {
if pod_Pod_Matcher_0.Match(i) { if pod_Pod_Matcher_0.Match(i) {
return "Pod", true return []string{"Pod"}
} }
return "Perl", true return []string{"Perl"}
}, },
".pro": func(i []byte) (string, bool) { ".pro": func(i []byte) []string {
if pro_Prolog_Matcher_0.Match(i) { if pro_Prolog_Matcher_0.Match(i) {
return "Prolog", true return []string{"Prolog"}
} else if pro_INI_Matcher_0.Match(i) { } else if pro_INI_Matcher_0.Match(i) {
return "INI", true return []string{"INI"}
} else if pro_QMake_Matcher_0.Match(i) && pro_QMake_Matcher_1.Match(i) { } else if pro_QMake_Matcher_0.Match(i) && pro_QMake_Matcher_1.Match(i) {
return "QMake", true return []string{"QMake"}
} else if pro_IDL_Matcher_0.Match(i) { } else if pro_IDL_Matcher_0.Match(i) {
return "IDL", true return []string{"IDL"}
} }
return OtherLanguage, false return nil
}, },
".props": func(i []byte) (string, bool) { ".props": func(i []byte) []string {
if props_XML_Matcher_0.Match(i) { if props_XML_Matcher_0.Match(i) {
return "XML", true return []string{"XML"}
} else if props_INI_Matcher_0.Match(i) { } else if props_INI_Matcher_0.Match(i) {
return "INI", true return []string{"INI"}
} }
return OtherLanguage, false return nil
}, },
".r": func(i []byte) (string, bool) { ".r": func(i []byte) []string {
if r_Rebol_Matcher_0.Match(i) { if r_Rebol_Matcher_0.Match(i) {
return "Rebol", true return []string{"Rebol"}
} else if r_R_Matcher_0.Match(i) { } else if r_R_Matcher_0.Match(i) {
return "R", true return []string{"R"}
} }
return OtherLanguage, false return nil
}, },
".rno": func(i []byte) (string, bool) { ".rno": func(i []byte) []string {
if rno_RUNOFF_Matcher_0.Match(i) { if rno_RUNOFF_Matcher_0.Match(i) {
return "RUNOFF", true return []string{"RUNOFF"}
} else if rno_Roff_Matcher_0.Match(i) { } else if rno_Roff_Matcher_0.Match(i) {
return "Roff", true return []string{"Roff"}
} }
return OtherLanguage, false return nil
}, },
".rpy": func(i []byte) (string, bool) { ".rpy": func(i []byte) []string {
if rpy_Python_Matcher_0.Match(i) { if rpy_Python_Matcher_0.Match(i) {
return "Python", true return []string{"Python"}
} }
return "Ren'Py", true return []string{"Ren'Py"}
}, },
".rs": func(i []byte) (string, bool) { ".rs": func(i []byte) []string {
if rs_Rust_Matcher_0.Match(i) { if rs_Rust_Matcher_0.Match(i) {
return "Rust", true return []string{"Rust"}
} else if rs_RenderScript_Matcher_0.Match(i) { } else if rs_RenderScript_Matcher_0.Match(i) {
return "RenderScript", true return []string{"RenderScript"}
} }
return OtherLanguage, false return nil
}, },
".sc": func(i []byte) (string, bool) { ".sc": func(i []byte) []string {
if sc_SuperCollider_Matcher_0.Match(i) || sc_SuperCollider_Matcher_1.Match(i) || sc_SuperCollider_Matcher_2.Match(i) { if sc_SuperCollider_Matcher_0.Match(i) || sc_SuperCollider_Matcher_1.Match(i) || sc_SuperCollider_Matcher_2.Match(i) {
return "SuperCollider", true return []string{"SuperCollider"}
} else if sc_Scala_Matcher_0.Match(i) || sc_Scala_Matcher_1.Match(i) || sc_Scala_Matcher_2.Match(i) { } else if sc_Scala_Matcher_0.Match(i) || sc_Scala_Matcher_1.Match(i) || sc_Scala_Matcher_2.Match(i) {
return "Scala", true return []string{"Scala"}
} }
return OtherLanguage, false return nil
}, },
".sql": func(i []byte) (string, bool) { ".sql": func(i []byte) []string {
if sql_PLpgSQL_Matcher_0.Match(i) || sql_PLpgSQL_Matcher_1.Match(i) || sql_PLpgSQL_Matcher_2.Match(i) { if sql_PLpgSQL_Matcher_0.Match(i) || sql_PLpgSQL_Matcher_1.Match(i) || sql_PLpgSQL_Matcher_2.Match(i) {
return "PLpgSQL", true return []string{"PLpgSQL"}
} else if sql_SQLPL_Matcher_0.Match(i) || sql_SQLPL_Matcher_1.Match(i) { } else if sql_SQLPL_Matcher_0.Match(i) || sql_SQLPL_Matcher_1.Match(i) {
return "SQLPL", true return []string{"SQLPL"}
} else if sql_PLSQL_Matcher_0.Match(i) || sql_PLSQL_Matcher_1.Match(i) { } else if sql_PLSQL_Matcher_0.Match(i) || sql_PLSQL_Matcher_1.Match(i) {
return "PLSQL", true return []string{"PLSQL"}
} else if sql_SQL_Matcher_0.Match(i) { } else if sql_SQL_Matcher_0.Match(i) {
return "SQL", true return []string{"SQL"}
} }
return OtherLanguage, false return nil
}, },
".srt": func(i []byte) (string, bool) { ".srt": func(i []byte) []string {
if srt_SubRipText_Matcher_0.Match(i) { if srt_SubRipText_Matcher_0.Match(i) {
return "SubRip Text", true return []string{"SubRip Text"}
} }
return OtherLanguage, false return nil
}, },
".t": func(i []byte) (string, bool) { ".t": func(i []byte) []string {
if t_Turing_Matcher_0.Match(i) { if t_Turing_Matcher_0.Match(i) {
return "Turing", true return []string{"Turing"}
} else if t_Perl6_Matcher_0.Match(i) { } else if t_Perl6_Matcher_0.Match(i) {
return "Perl6", true return []string{"Perl6"}
} else if t_Perl_Matcher_0.Match(i) { } else if t_Perl_Matcher_0.Match(i) {
return "Perl", true return []string{"Perl"}
} }
return OtherLanguage, false return nil
}, },
".toc": func(i []byte) (string, bool) { ".toc": func(i []byte) []string {
if toc_WorldofWarcraftAddonData_Matcher_0.Match(i) { if toc_WorldofWarcraftAddonData_Matcher_0.Match(i) {
return "World of Warcraft Addon Data", true return []string{"World of Warcraft Addon Data"}
} else if toc_TeX_Matcher_0.Match(i) { } else if toc_TeX_Matcher_0.Match(i) {
return "TeX", true return []string{"TeX"}
} }
return OtherLanguage, false return nil
}, },
".ts": func(i []byte) (string, bool) { ".ts": func(i []byte) []string {
if ts_XML_Matcher_0.Match(i) { if ts_XML_Matcher_0.Match(i) {
return "XML", true return []string{"XML"}
} }
return "TypeScript", true return []string{"TypeScript"}
}, },
".tst": func(i []byte) (string, bool) { ".tst": func(i []byte) []string {
if tst_GAP_Matcher_0.Match(i) { if tst_GAP_Matcher_0.Match(i) {
return "GAP", true return []string{"GAP"}
} }
return "Scilab", true return []string{"Scilab"}
}, },
".tsx": func(i []byte) (string, bool) { ".tsx": func(i []byte) []string {
if tsx_TypeScript_Matcher_0.Match(i) { if tsx_TypeScript_Matcher_0.Match(i) {
return "TypeScript", true return []string{"TypeScript"}
} else if tsx_XML_Matcher_0.Match(i) { } else if tsx_XML_Matcher_0.Match(i) {
return "XML", true return []string{"XML"}
} }
return OtherLanguage, false return nil
}, },
} }

65
content_test.go
View File

@ -1,65 +0,0 @@
package slinguist
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"text/tabwriter"
. "gopkg.in/check.v1"
)
func (s *TSuite) TestGetLanguageByContentLinguistCorpus(c *C) {
var total, failed, ok, other, unsafe int
w := new(tabwriter.Writer)
w.Init(os.Stdout, 0, 8, 0, '\t', 0)
filepath.Walk(".linguist/samples", func(path string, f os.FileInfo, err error) error {
if f.IsDir() {
if f.Name() == "filenames" {
return filepath.SkipDir
}
return nil
}
expected := filepath.Base(filepath.Dir(path))
filename := filepath.Base(path)
extension := filepath.Ext(path)
content, _ := ioutil.ReadFile(path)
if extension == "" {
return nil
}
total++
obtained, safe := GetLanguageByContent(filename, content)
if obtained == OtherLanguage {
other++
}
var status string
if expected == obtained {
status = "ok"
ok++
} else {
status = "failed"
failed++
if !safe {
unsafe++
}
}
fmt.Fprintf(w, "%s\t%s\t%s\t%v\t%s\n", filename, expected, obtained, safe, status)
return nil
})
fmt.Fprintln(w)
w.Flush()
fmt.Printf("total files: %d, ok: %d, failed: %d, unsafe: %d, other: %d\n", total, ok, failed, unsafe, other)
}

2
documentation_matchers.go → documentation.go
View File

@ -2,7 +2,7 @@ package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND // THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: 60f864a138650dd17fafc94814be9ee2d3aaef8c // Extracted from github/linguist commit: b6460f8ed6b249281ada099ca28bd8f1230b8892
import "gopkg.in/toqueteos/substring.v1" import "gopkg.in/toqueteos/substring.v1"

1459
extension.go
View File

File diff suppressed because it is too large Load Diff

22
extension_test.go
View File

@ -1,22 +0,0 @@
package slinguist
import . "gopkg.in/check.v1"
func (s *TSuite) TestGetLanguageByExtension(c *C) {
lang, safe := GetLanguageByExtension("foo.foo")
c.Assert(lang, Equals, "Other")
c.Assert(safe, Equals, false)
lang, safe = GetLanguageByExtension("foo.go")
c.Assert(lang, Equals, "Go")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByExtension("foo.go.php")
c.Assert(lang, Equals, "Hack")
c.Assert(safe, Equals, false)
}
func (s *TSuite) TestGetLanguageExtensions(c *C) {
c.Assert(GetLanguageExtensions("foo"), HasLen, 0)
c.Assert(GetLanguageExtensions("C"), Not(HasLen), 0)
}

1004
extensions_map.go
View File

File diff suppressed because it is too large Load Diff

145
filename.go
View File

@ -1,10 +1,143 @@
package slinguist package slinguist
func GetLanguageByFilename(filename string) (lang string, safe bool) { // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
lang, safe = languagesByFilename[filename] // THIS FILE SHOULD NOT BE EDITED BY HAND
if lang == "" { // Extracted from github/linguist commit: b6460f8ed6b249281ada099ca28bd8f1230b8892
lang = OtherLanguage
}
return var languagesByFilename = map[string][]string{
".Rprofile": {"R"},
".XCompose": {"XCompose"},
".abbrev_defs": {"Emacs Lisp"},
".arcconfig": {"JSON"},
".babelrc": {"JSON5"},
".bash_history": {"Shell"},
".bash_logout": {"Shell"},
".bash_profile": {"Shell"},
".bashrc": {"Shell"},
".clang-format": {"YAML"},
".classpath": {"XML"},
".emacs": {"Emacs Lisp"},
".emacs.desktop": {"Emacs Lisp"},
".factor-boot-rc": {"Factor"},
".factor-rc": {"Factor"},
".gclient": {"Python"},
".gnus": {"Emacs Lisp"},
".irbrc": {"Ruby"},
".jshintrc": {"JSON"},
".nvimrc": {"Vim script"},
".php_cs": {"PHP"},
".php_cs.dist": {"PHP"},
".project": {"XML"},
".pryrc": {"Ruby"},
".spacemacs": {"Emacs Lisp"},
".vimrc": {"Vim script"},
".viper": {"Emacs Lisp"},
"APKBUILD": {"Alpine Abuild"},
"App.config": {"XML"},
"Appraisals": {"Ruby"},
"BSDmakefile": {"Makefile"},
"BUCK": {"Python"},
"BUILD": {"Python"},
"Berksfile": {"Ruby"},
"Brewfile": {"Ruby"},
"Buildfile": {"Ruby"},
"CMakeLists.txt": {"CMake"},
"COPYING": {"Text"},
"COPYRIGHT.regex": {"Text"},
"Cakefile": {"CoffeeScript"},
"Cask": {"Emacs Lisp"},
"Dangerfile": {"Ruby"},
"Deliverfile": {"Ruby"},
"Dockerfile": {"Dockerfile"},
"Emakefile": {"Erlang"},
"FONTLOG": {"Text"},
"Fakefile": {"Fancy"},
"Fastfile": {"Ruby"},
"GNUmakefile": {"Makefile"},
"Gemfile": {"Ruby"},
"Gemfile.lock": {"Ruby"},
"Guardfile": {"Ruby"},
"INSTALL": {"Text"},
"INSTALL.mysql": {"Text"},
"Jakefile": {"JavaScript"},
"Jarfile": {"Ruby"},
"Jenkinsfile": {"Groovy"},
"Kbuild": {"Makefile"},
"LICENSE": {"Text"},
"LICENSE.mysql": {"Text"},
"Makefile": {"Makefile"},
"Makefile.am": {"Makefile"},
"Makefile.boot": {"Makefile"},
"Makefile.frag": {"Makefile"},
"Makefile.in": {"Makefile"},
"Makefile.inc": {"Makefile"},
"Makefile.wat": {"Makefile"},
"Mavenfile": {"Ruby"},
"Modulefile": {"Puppet"},
"NEWS": {"Text"},
"Notebook": {"Jupyter Notebook"},
"NuGet.config": {"XML"},
"Nukefile": {"Nu"},
"PKGBUILD": {"Shell"},
"Phakefile": {"PHP"},
"Podfile": {"Ruby"},
"Project.ede": {"Emacs Lisp"},
"Puppetfile": {"Ruby"},
"README.1ST": {"Text"},
"README.me": {"Text"},
"README.mysql": {"Text"},
"ROOT": {"Isabelle ROOT"},
"Rakefile": {"Ruby"},
"Rexfile": {"Perl6"},
"SConscript": {"Python"},
"SConstruct": {"Python"},
"Settings.StyleCop": {"XML"},
"Slakefile": {"LiveScript"},
"Snakefile": {"Python"},
"Snapfile": {"Ruby"},
"Thorfile": {"Ruby"},
"Vagrantfile": {"Ruby"},
"WORKSPACE": {"Python"},
"Web.Debug.config": {"XML"},
"Web.Release.config": {"XML"},
"Web.config": {"XML"},
"XCompose": {"XCompose"},
"_emacs": {"Emacs Lisp"},
"_vimrc": {"Vim script"},
"abbrev_defs": {"Emacs Lisp"},
"ant.xml": {"Ant Build System"},
"build.xml": {"Ant Build System"},
"buildfile": {"Ruby"},
"click.me": {"Text"},
"composer.lock": {"JSON"},
"configure.ac": {"M4Sugar"},
"delete.me": {"Text"},
"descrip.mmk": {"Module Management System"},
"descrip.mms": {"Module Management System"},
"gradlew": {"Shell"},
"gvimrc": {"Vim script"},
"keep.me": {"Text"},
"ld.script": {"Linker Script"},
"makefile": {"Makefile"},
"makefile.sco": {"Makefile"},
"mcmod.info": {"JSON"},
"meson.build": {"Meson"},
"meson_options.txt": {"Meson"},
"mix.lock": {"Elixir"},
"mkfile": {"Makefile"},
"mmn": {"Roff"},
"mmt": {"Roff"},
"nginx.conf": {"Nginx"},
"nvimrc": {"Vim script"},
"packages.config": {"XML"},
"pom.xml": {"Maven POM"},
"read.me": {"Text"},
"rebar.config": {"Erlang"},
"rebar.config.lock": {"Erlang"},
"rebar.lock": {"Erlang"},
"riemann.config": {"Clojure"},
"test.me": {"Text"},
"vimrc": {"Vim script"},
"wscript": {"Python"},
"xcompose": {"XCompose"},
} }

37
filename_test.go
View File

@ -1,37 +0,0 @@
package slinguist
import . "gopkg.in/check.v1"
func (s *TSuite) TestGetLanguageByFilename(c *C) {
lang, safe := GetLanguageByFilename(`unknown.interpreter`)
c.Assert(lang, Equals, OtherLanguage)
c.Assert(safe, Equals, false)
lang, safe = GetLanguageByFilename(`.bashrc`)
c.Assert(lang, Equals, "Shell")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByFilename(`Dockerfile`)
c.Assert(lang, Equals, "Dockerfile")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByFilename(`Makefile.frag`)
c.Assert(lang, Equals, "Makefile")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByFilename(`makefile`)
c.Assert(lang, Equals, "Makefile")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByFilename(`Vagrantfile`)
c.Assert(lang, Equals, "Ruby")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByFilename(`_vimrc`)
c.Assert(lang, Equals, "Vim script")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByFilename(`pom.xml`)
c.Assert(lang, Equals, "Maven POM")
c.Assert(safe, Equals, true)
}

142
filenames_map.go
View File

@ -1,142 +0,0 @@
package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: 60f864a138650dd17fafc94814be9ee2d3aaef8c
var languagesByFilename = map[string]string{
".Rprofile": "R",
".XCompose": "XCompose",
".abbrev_defs": "Emacs Lisp",
".arcconfig": "JSON",
".babelrc": "JSON5",
".bash_history": "Shell",
".bash_logout": "Shell",
".bash_profile": "Shell",
".bashrc": "Shell",
".clang-format": "YAML",
".classpath": "XML",
".emacs": "Emacs Lisp",
".emacs.desktop": "Emacs Lisp",
".factor-boot-rc": "Factor",
".factor-rc": "Factor",
".gclient": "Python",
".gnus": "Emacs Lisp",
".irbrc": "Ruby",
".jshintrc": "JSON",
".nvimrc": "Vim script",
".php_cs": "PHP",
".php_cs.dist": "PHP",
".project": "XML",
".pryrc": "Ruby",
".spacemacs": "Emacs Lisp",
".vimrc": "Vim script",
".viper": "Emacs Lisp",
"APKBUILD": "Alpine Abuild",
"App.config": "XML",
"Appraisals": "Ruby",
"BSDmakefile": "Makefile",
"BUCK": "Python",
"BUILD": "Python",
"Berksfile": "Ruby",
"Brewfile": "Ruby",
"Buildfile": "Ruby",
"CMakeLists.txt": "CMake",
"COPYING": "Text",
"COPYRIGHT.regex": "Text",
"Cakefile": "CoffeeScript",
"Cask": "Emacs Lisp",
"Dangerfile": "Ruby",
"Deliverfile": "Ruby",
"Dockerfile": "Dockerfile",
"Emakefile": "Erlang",
"FONTLOG": "Text",
"Fakefile": "Fancy",
"Fastfile": "Ruby",
"GNUmakefile": "Makefile",
"Gemfile": "Ruby",
"Gemfile.lock": "Ruby",
"Guardfile": "Ruby",
"INSTALL": "Text",
"INSTALL.mysql": "Text",
"Jakefile": "JavaScript",
"Jarfile": "Ruby",
"Jenkinsfile": "Groovy",
"Kbuild": "Makefile",
"LICENSE": "Text",
"LICENSE.mysql": "Text",
"Makefile": "Makefile",
"Makefile.am": "Makefile",
"Makefile.boot": "Makefile",
"Makefile.frag": "Makefile",
"Makefile.in": "Makefile",
"Makefile.inc": "Makefile",
"Mavenfile": "Ruby",
"Modulefile": "Puppet",
"NEWS": "Text",
"Notebook": "Jupyter Notebook",
"NuGet.config": "XML",
"Nukefile": "Nu",
"PKGBUILD": "Shell",
"Phakefile": "PHP",
"Podfile": "Ruby",
"Project.ede": "Emacs Lisp",
"Puppetfile": "Ruby",
"README.1ST": "Text",
"README.me": "Text",
"README.mysql": "Text",
"ROOT": "Isabelle ROOT",
"Rakefile": "Ruby",
"Rexfile": "Perl6",
"SConscript": "Python",
"SConstruct": "Python",
"Settings.StyleCop": "XML",
"Slakefile": "LiveScript",
"Snakefile": "Python",
"Snapfile": "Ruby",
"Thorfile": "Ruby",
"Vagrantfile": "Ruby",
"WORKSPACE": "Python",
"Web.Debug.config": "XML",
"Web.Release.config": "XML",
"Web.config": "XML",
"XCompose": "XCompose",
"_emacs": "Emacs Lisp",
"_vimrc": "Vim script",
"abbrev_defs": "Emacs Lisp",
"ant.xml": "Ant Build System",
"build.xml": "Ant Build System",
"buildfile": "Ruby",
"click.me": "Text",
"composer.lock": "JSON",
"configure.ac": "M4Sugar",
"delete.me": "Text",
"descrip.mmk": "Module Management System",
"descrip.mms": "Module Management System",
"gradlew": "Shell",
"gvimrc": "Vim script",
"keep.me": "Text",
"ld.script": "Linker Script",
"makefile": "Makefile",
"makefile.sco": "Makefile",
"mcmod.info": "JSON",
"meson.build": "Meson",
"meson_options.txt": "Meson",
"mix.lock": "Elixir",
"mkfile": "Makefile",
"mmn": "Roff",
"mmt": "Roff",
"nginx.conf": "Nginx",
"nvimrc": "Vim script",
"packages.config": "XML",
"pom.xml": "Maven POM",
"read.me": "Text",
"rebar.config": "Erlang",
"rebar.config.lock": "Erlang",
"rebar.lock": "Erlang",
"riemann.config": "Clojure",
"test.me": "Text",
"vimrc": "Vim script",
"wscript": "Python",
"xcompose": "XCompose",
}

76219
frequencies.go
View File

File diff suppressed because it is too large Load Diff

4
internal/code-generator/assets/aliases.go.tmpl
View File

@ -4,8 +4,8 @@ package slinguist
// THIS FILE SHOULD NOT BE EDITED BY HAND // THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: {{ getCommit }} // Extracted from github/linguist commit: {{ getCommit }}
// languagesByAlias keeps alias for different languages and use the name of the languages as a alias too. All the // languagesByAlias keeps alias for different languages and use the name of the languages as an alias too.
// keys (alias or not) are written in lower case and the whitespaces has been replaced by underscores. // All the keys (alias or not) are written in lower case and the whitespaces has been replaced by underscores.
var languagesByAlias = map[string]string{ var languagesByAlias = map[string]string{
{{range $alias, $language := . -}} {{range $alias, $language := . -}}
"{{ $alias }}": {{ printf "%q" $language -}}, "{{ $alias }}": {{ printf "%q" $language -}},

23
internal/code-generator/assets/content.go.tmpl
View File

@ -5,27 +5,14 @@ package slinguist
// Extracted from github/linguist commit: {{ getCommit }} // Extracted from github/linguist commit: {{ getCommit }}
import ( import (
"path/filepath"
"regexp" "regexp"
"strings"
) )
func GetLanguageByContent(filename string, content []byte) (lang string, safe bool) { type languageMatcher func ([]byte) []string
ext := strings.ToLower(filepath.Ext(filename))
if fnMatcher, ok := matchers[ext]; ok {
lang, safe = fnMatcher(content)
} else {
lang = OtherLanguage
}
return var contentMatchers = map[string]languageMatcher{
}
type languageMatcher func ([]byte) (string, bool)
var matchers = map[string]languageMatcher{
{{ range $index, $disambiguator := . -}} {{ range $index, $disambiguator := . -}}
{{ printf "%q" $disambiguator.Extension }}: func(i []byte) (string, bool) { {{ printf "%q" $disambiguator.Extension }}: func(i []byte) []string {
{{ range $i, $language := $disambiguator.Languages -}} {{ range $i, $language := $disambiguator.Languages -}}
{{- if not (avoidLanguage $language) }} {{- if not (avoidLanguage $language) }}
@ -33,14 +20,14 @@ var matchers = map[string]languageMatcher{
{{- if gt $i 0 }} else {{ end -}} {{- if gt $i 0 }} else {{ end -}}
if {{- range $j, $heuristic := $language.Heuristics }} {{ $heuristic.Name }}.Match(i) if {{- range $j, $heuristic := $language.Heuristics }} {{ $heuristic.Name }}.Match(i)
{{- if lt $j (len $language.LogicRelations) }} {{index $language.LogicRelations $j}} {{- end -}} {{ end }} { {{- if lt $j (len $language.LogicRelations) }} {{index $language.LogicRelations $j}} {{- end -}} {{ end }} {
return {{ printf "%q" $language.Language }}, true return []string{ {{- printf "%q" $language.Language -}} }
} }
{{- end -}} {{- end -}}
{{- end -}} {{- end -}}
{{- end}} {{- end}}
return {{ returnLanguage $disambiguator.Languages }}, {{ safeLanguage $disambiguator.Languages }} return {{ returnLanguages $disambiguator.Languages | returnStringSlice }}
}, },
{{ end -}} {{ end -}}
} }

8
internal/code-generator/assets/extensions.go.tmpl
View File

@ -5,7 +5,13 @@ package slinguist
// Extracted from github/linguist commit: {{ getCommit }} // Extracted from github/linguist commit: {{ getCommit }}
var languagesByExtension = map[string][]string{ var languagesByExtension = map[string][]string{
{{range $extension, $languages := . -}} {{range $extension, $languages := .LanguagesByExtension -}}
"{{ $extension }}": { {{- $languages | formatStringSlice -}} }, "{{ $extension }}": { {{- $languages | formatStringSlice -}} },
{{end -}} {{end -}}
} }
var extensionsByLanguage = map[string][]string{
{{range $language, $extensions := .ExtensionsByLanguage -}}
"{{ $language }}": { {{- $extensions | formatStringSlice -}} },
{{end -}}
}

6
internal/code-generator/assets/filenames.go.tmpl
View File

@ -4,8 +4,8 @@ package slinguist
// THIS FILE SHOULD NOT BE EDITED BY HAND // THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: {{ getCommit }} // Extracted from github/linguist commit: {{ getCommit }}
var languagesByFilename = map[string]string{ var languagesByFilename = map[string][]string{
{{range $filename, $language := . -}} {{range $filename, $languages := . -}}
"{{ $filename }}": {{- printf "%q" $language -}}, "{{ $filename }}": { {{- formatStringSlice $languages -}} },
{{end -}} {{end -}}
} }

5
internal/code-generator/generator/aliases.go
View File

@ -3,14 +3,13 @@ package generator
import ( import (
"bytes" "bytes"
"io" "io"
"text/template"
"strings" "strings"
"text/template"
yaml "gopkg.in/yaml.v2" yaml "gopkg.in/yaml.v2"
) )
// Aliases reads from buf and builds aliases_map.go file from aliasesTmplPath. // Aliases reads from buf and builds source file from aliasesTmplPath.
func Aliases(data []byte, aliasesTmplPath, aliasesTmplName, commit string) ([]byte, error) { func Aliases(data []byte, aliasesTmplPath, aliasesTmplName, commit string) ([]byte, error) {
languages := make(map[string]*languageInfo) languages := make(map[string]*languageInfo)
if err := yaml.Unmarshal(data, &languages); err != nil { if err := yaml.Unmarshal(data, &languages); err != nil {

4
internal/code-generator/generator/documentation.go
View File

@ -2,13 +2,13 @@ package generator
import ( import (
"bytes" "bytes"
"html/template"
"io" "io"
"text/template"
yaml "gopkg.in/yaml.v2" yaml "gopkg.in/yaml.v2"
) )
// Documentation reads from buf and builds documentation_matchers.go file from documentationTmplPath. // Documentation reads from buf and builds source file from documentationTmplPath.
func Documentation(data []byte, documentationTmplPath, documentationTmplName, commit string) ([]byte, error) { func Documentation(data []byte, documentationTmplPath, documentationTmplName, commit string) ([]byte, error) {
var regexpList []string var regexpList []string
if err := yaml.Unmarshal(data, &regexpList); err != nil { if err := yaml.Unmarshal(data, &regexpList); err != nil {

28
internal/code-generator/generator/extensions.go
View File

@ -9,18 +9,25 @@ import (
yaml "gopkg.in/yaml.v2" yaml "gopkg.in/yaml.v2"
) )
// Extensions reads from buf and builds extensions_map.go file from extensionsTmplPath. type extensionsInfo struct {
LanguagesByExtension map[string][]string
ExtensionsByLanguage map[string][]string
}
// Extensions reads from buf and builds source file from extensionsTmplPath.
func Extensions(data []byte, extensionsTmplPath, extensionsTmplName, commit string) ([]byte, error) { func Extensions(data []byte, extensionsTmplPath, extensionsTmplName, commit string) ([]byte, error) {
languages := make(map[string]*languageInfo) languages := make(map[string]*languageInfo)
if err := yaml.Unmarshal(data, &languages); err != nil { if err := yaml.Unmarshal(data, &languages); err != nil {
return nil, err return nil, err
} }
extInfo := &extensionsInfo{}
orderedKeyList := getAlphabeticalOrderedKeys(languages) orderedKeyList := getAlphabeticalOrderedKeys(languages)
languagesByExtension := buildExtensionLanguageMap(languages, orderedKeyList) extInfo.LanguagesByExtension = buildExtensionLanguageMap(languages, orderedKeyList)
extInfo.ExtensionsByLanguage = buildLanguageExtensionsMap(languages)
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
if err := executeExtensionsTemplate(buf, languagesByExtension, extensionsTmplPath, extensionsTmplName, commit); err != nil { if err := executeExtensionsTemplate(buf, extInfo, extensionsTmplPath, extensionsTmplName, commit); err != nil {
return nil, err return nil, err
} }
@ -39,14 +46,25 @@ func buildExtensionLanguageMap(languages map[string]*languageInfo, orderedKeyLis
return extensionLangsMap return extensionLangsMap
} }
func executeExtensionsTemplate(out io.Writer, languagesByExtension map[string][]string, extensionsTmplPath, extensionsTmpl, commit string) error { func buildLanguageExtensionsMap(languages map[string]*languageInfo) map[string][]string {
langExtensionMap := make(map[string][]string, len(languages))
for lang, info := range languages {
if len(info.Extensions) > 0 {
langExtensionMap[lang] = info.Extensions
}
}
return langExtensionMap
}
func executeExtensionsTemplate(out io.Writer, extInfo *extensionsInfo, extensionsTmplPath, extensionsTmpl, commit string) error {
fmap := template.FuncMap{ fmap := template.FuncMap{
"getCommit": func() string { return commit }, "getCommit": func() string { return commit },
"formatStringSlice": func(slice []string) string { return `"` + strings.Join(slice, `","`) + `"` }, "formatStringSlice": func(slice []string) string { return `"` + strings.Join(slice, `","`) + `"` },
} }
t := template.Must(template.New(extensionsTmpl).Funcs(fmap).ParseFiles(extensionsTmplPath)) t := template.Must(template.New(extensionsTmpl).Funcs(fmap).ParseFiles(extensionsTmplPath))
if err := t.Execute(out, languagesByExtension); err != nil { if err := t.Execute(out, extInfo); err != nil {
return err return err
} }

12
internal/code-generator/generator/filenames.go
View File

@ -3,12 +3,13 @@ package generator
import ( import (
"bytes" "bytes"
"io" "io"
"strings"
"text/template" "text/template"
yaml "gopkg.in/yaml.v2" yaml "gopkg.in/yaml.v2"
) )
// Filenames reads from buf and builds filenames_map.go file from filenamesTmplPath. // Filenames reads from buf and builds source file from filenamesTmplPath.
func Filenames(data []byte, filenamesTmplPath, filenamesTmplName, commit string) ([]byte, error) { func Filenames(data []byte, filenamesTmplPath, filenamesTmplName, commit string) ([]byte, error) {
languages := make(map[string]*languageInfo) languages := make(map[string]*languageInfo)
if err := yaml.Unmarshal(data, &languages); err != nil { if err := yaml.Unmarshal(data, &languages); err != nil {
@ -25,20 +26,21 @@ func Filenames(data []byte, filenamesTmplPath, filenamesTmplName, commit string)
return buf.Bytes(), nil return buf.Bytes(), nil
} }
func buildFilenameLanguageMap(languages map[string]*languageInfo) map[string]string { func buildFilenameLanguageMap(languages map[string]*languageInfo) map[string][]string {
filenameLangMap := make(map[string]string) filenameLangMap := make(map[string][]string)
for lang, langInfo := range languages { for lang, langInfo := range languages {
for _, filename := range langInfo.Filenames { for _, filename := range langInfo.Filenames {
filenameLangMap[filename] = lang filenameLangMap[filename] = append(filenameLangMap[filename], lang)
} }
} }
return filenameLangMap return filenameLangMap
} }
func executeFilenamesTemplate(out io.Writer, languagesByFilename map[string]string, filenamesTmplPath, filenamesTmpl, commit string) error { func executeFilenamesTemplate(out io.Writer, languagesByFilename map[string][]string, filenamesTmplPath, filenamesTmpl, commit string) error {
fmap := template.FuncMap{ fmap := template.FuncMap{
"getCommit": func() string { return commit }, "getCommit": func() string { return commit },
"formatStringSlice": func(slice []string) string { return `"` + strings.Join(slice, `","`) + `"` },
} }
t := template.Must(template.New(filenamesTmpl).Funcs(fmap).ParseFiles(filenamesTmplPath)) t := template.Must(template.New(filenamesTmpl).Funcs(fmap).ParseFiles(filenamesTmplPath))

2
internal/code-generator/generator/generator.go
View File

@ -5,7 +5,7 @@ import (
"io/ioutil" "io/ioutil"
) )
// Func is the function's type that generate the files from templates. // Func is the function's type that generate source file from a data to be parsed and a template.
type Func func(dataToParse []byte, templatePath string, template string, commit string) ([]byte, error) type Func func(dataToParse []byte, templatePath string, template string, commit string) ([]byte, error)
// FromFile read data to parse from a file named fileToParse and write the generated source code to a file named outPath. The generated // FromFile read data to parse from a file named fileToParse and write the generated source code to a file named outPath. The generated

16
internal/code-generator/generator/generator_test.go
View File

@ -15,7 +15,7 @@ import (
const ( const (
lingustURL = "https://github.com/github/linguist.git" lingustURL = "https://github.com/github/linguist.git"
commitTree = "60f864a138650dd17fafc94814be9ee2d3aaef8c" commitTree = "60f864a138650dd17fafc94814be9ee2d3aaef8c"
commitTest = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7" commitTest = "0123456789abcdef0123456789abcdef01234567"
// Extensions test // Extensions test
extensionsTestFile = "test_files/extensions.test.yml" extensionsTestFile = "test_files/extensions.test.yml"
@ -77,6 +77,10 @@ type GeneratorTestSuite struct {
tmpLinguist string tmpLinguist string
} }
func TestGeneratorTestSuite(t *testing.T) {
suite.Run(t, new(GeneratorTestSuite))
}
func (g *GeneratorTestSuite) SetupSuite() { func (g *GeneratorTestSuite) SetupSuite() {
tmpLinguist, err := ioutil.TempDir("", "linguist-") tmpLinguist, err := ioutil.TempDir("", "linguist-")
assert.NoError(g.T(), err) assert.NoError(g.T(), err)
@ -116,7 +120,7 @@ func (g *GeneratorTestSuite) TestFromFile() {
wantOut string wantOut string
}{ }{
{ {
name: "TestFromFile_Language", name: "TestFromFile_Extensions",
fileToParse: extensionsTestFile, fileToParse: extensionsTestFile,
tmplPath: extensionsTestTmplPath, tmplPath: extensionsTestTmplPath,
tmplName: extensionsTestTmplName, tmplName: extensionsTestTmplName,
@ -201,7 +205,7 @@ func (g *GeneratorTestSuite) TestFromFile() {
assert.NoError(g.T(), err) assert.NoError(g.T(), err)
out, err := ioutil.ReadFile(outPath.Name()) out, err := ioutil.ReadFile(outPath.Name())
assert.NoError(g.T(), err) assert.NoError(g.T(), err)
assert.EqualValues(g.T(), gold, out, fmt.Sprintf("FromFile() = %v, want %v", string(out), string(test.wantOut))) assert.EqualValues(g.T(), gold, out, fmt.Sprintf("%v: %v, expected: %v", test.name, string(out), string(test.wantOut)))
} }
} }
@ -236,10 +240,6 @@ func (g *GeneratorTestSuite) TestFrequencies() {
assert.NoError(g.T(), err) assert.NoError(g.T(), err)
out, err := ioutil.ReadFile(outPath.Name()) out, err := ioutil.ReadFile(outPath.Name())
assert.NoError(g.T(), err) assert.NoError(g.T(), err)
assert.EqualValues(g.T(), gold, out, fmt.Sprintf("Frequencies() = %v, want %v", string(out), string(test.wantOut))) assert.EqualValues(g.T(), gold, out, fmt.Sprintf("%v: %v, expected: %v", test.name, string(out), string(test.wantOut)))
} }
} }
func TestGeneratorTestSuite(t *testing.T) {
suite.Run(t, new(GeneratorTestSuite))
}

36
internal/code-generator/generator/heuristics.go
View File

@ -11,7 +11,7 @@ import (
"text/template" "text/template"
) )
// Heuristics reads from buf and builds content.go file from contentTmplPath. // Heuristics reads from buf and builds source file from contentTmplPath.
func Heuristics(heuristics []byte, contentTmplPath, contentTmplName, commit string) ([]byte, error) { func Heuristics(heuristics []byte, contentTmplPath, contentTmplName, commit string) ([]byte, error) {
disambiguators, err := getDisambiguators(heuristics) disambiguators, err := getDisambiguators(heuristics)
if err != nil { if err != nil {
@ -24,6 +24,8 @@ func Heuristics(heuristics []byte, contentTmplPath, contentTmplName, commit stri
} }
return buf.Bytes(), nil return buf.Bytes(), nil
// fmt.Println(string(buf.Bytes()))
// return nil, nil
} }
const unknownLanguage = "OtherLanguage" const unknownLanguage = "OtherLanguage"
@ -417,8 +419,14 @@ func executeContentTemplate(out io.Writer, disambiguators []*disambiguator, cont
fmap := template.FuncMap{ fmap := template.FuncMap{
"getCommit": func() string { return commit }, "getCommit": func() string { return commit },
"getAllHeuristics": getAllHeuristics, "getAllHeuristics": getAllHeuristics,
"returnLanguage": returnLanguage, "returnStringSlice": func(slice []string) string {
"safeLanguage": safeLanguage, if len(slice) == 0 {
return "nil"
}
return `[]string{` + strings.Join(slice, `, `) + `}`
},
"returnLanguages": returnLanguages,
"avoidLanguage": avoidLanguage, "avoidLanguage": avoidLanguage,
} }
@ -458,18 +466,7 @@ func containsInvalidRegexp(reg string) bool {
return strings.Contains(reg, `(?<`) || strings.Contains(reg, `\1`) return strings.Contains(reg, `(?<`) || strings.Contains(reg, `\1`)
} }
func returnLanguage(langsHeuristics []*languageHeuristics) string { func returnLanguages(langsHeuristics []*languageHeuristics) []string {
lang, _ := returnLangAndSafe(langsHeuristics)
return lang
}
func safeLanguage(langsHeuristics []*languageHeuristics) bool {
_, safe := returnLangAndSafe(langsHeuristics)
return safe
}
func returnLangAndSafe(langsHeuristics []*languageHeuristics) (string, bool) {
// at the moment, only returns one string although might be exists several language to return as a []string.
langs := make([]string, 0) langs := make([]string, 0)
for _, langHeu := range langsHeuristics { for _, langHeu := range langsHeuristics {
if len(langHeu.Heuristics) == 0 { if len(langHeu.Heuristics) == 0 {
@ -477,12 +474,5 @@ func returnLangAndSafe(langsHeuristics []*languageHeuristics) (string, bool) {
} }
} }
lang := unknownLanguage return langs
safe := false
if len(langs) != 0 {
lang = langs[0]
safe = len(langs) == 1
}
return lang, safe
} }

2
internal/code-generator/generator/interpreters.go
View File

@ -9,7 +9,7 @@ import (
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
) )
// Interpreters reads from buf and builds interpreters_map.go file from interpretersTmplPath. // Interpreters reads from buf and builds source file from interpretersTmplPath.
func Interpreters(data []byte, interpretersTmplPath, interpretersTmplName, commit string) ([]byte, error) { func Interpreters(data []byte, interpretersTmplPath, interpretersTmplName, commit string) ([]byte, error) {
languages := make(map[string]*languageInfo) languages := make(map[string]*languageInfo)
if err := yaml.Unmarshal(data, &languages); err != nil { if err := yaml.Unmarshal(data, &languages); err != nil {

6
internal/code-generator/generator/test_files/aliases.gold
View File

@ -2,10 +2,10 @@ package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND // THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7 // Extracted from github/linguist commit: 0123456789abcdef0123456789abcdef01234567
// languagesByAlias keeps alias for different languages and use the name of the languages as a alias too. All the // languagesByAlias keeps alias for different languages and use the name of the languages as an alias too.
// keys (alias or not) are written in lower case and the whitespaces has been replaced by underscores. // All the keys (alias or not) are written in lower case and the whitespaces has been replaced by underscores.
var languagesByAlias = map[string]string{ var languagesByAlias = map[string]string{
"asp": "ASP", "asp": "ASP",
"aspx": "ASP", "aspx": "ASP",

103
internal/code-generator/generator/test_files/content.gold
View File

@ -2,121 +2,108 @@ package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND // THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7 // Extracted from github/linguist commit: 0123456789abcdef0123456789abcdef01234567
import ( import (
"path/filepath"
"regexp" "regexp"
"strings"
) )
func GetLanguageByContent(filename string, content []byte) (lang string, safe bool) { type languageMatcher func([]byte) []string
ext := strings.ToLower(filepath.Ext(filename))
if fnMatcher, ok := matchers[ext]; ok {
lang, safe = fnMatcher(content)
} else {
lang = OtherLanguage
}
return var contentMatchers = map[string]languageMatcher{
} ".asc": func(i []byte) []string {
type languageMatcher func([]byte) (string, bool)
var matchers = map[string]languageMatcher{
".asc": func(i []byte) (string, bool) {
if asc_PublicKey_Matcher_0.Match(i) { if asc_PublicKey_Matcher_0.Match(i) {
return "Public Key", true return []string{"Public Key"}
} else if asc_AsciiDoc_Matcher_0.Match(i) { } else if asc_AsciiDoc_Matcher_0.Match(i) {
return "AsciiDoc", true return []string{"AsciiDoc"}
} else if asc_AGSScript_Matcher_0.Match(i) { } else if asc_AGSScript_Matcher_0.Match(i) {
return "AGS Script", true return []string{"AGS Script"}
} }
return OtherLanguage, false return nil
}, },
".f": func(i []byte) (string, bool) { ".f": func(i []byte) []string {
if f_Forth_Matcher_0.Match(i) { if f_Forth_Matcher_0.Match(i) {
return "Forth", true return []string{"Forth"}
} else if f_FilebenchWML_Matcher_0.Match(i) { } else if f_FilebenchWML_Matcher_0.Match(i) {
return "Filebench WML", true return []string{"Filebench WML"}
} else if f_FORTRAN_Matcher_0.Match(i) { } else if f_FORTRAN_Matcher_0.Match(i) {
return "FORTRAN", true return []string{"FORTRAN"}
} }
return OtherLanguage, false return nil
}, },
".h": func(i []byte) (string, bool) { ".h": func(i []byte) []string {
if h_ObjectiveDashC_Matcher_0.Match(i) { if h_ObjectiveDashC_Matcher_0.Match(i) {
return "Objective-C", true return []string{"Objective-C"}
} else if h_CPlusPlus_Matcher_0.Match(i) || h_CPlusPlus_Matcher_1.Match(i) || h_CPlusPlus_Matcher_2.Match(i) || h_CPlusPlus_Matcher_3.Match(i) || h_CPlusPlus_Matcher_4.Match(i) || h_CPlusPlus_Matcher_5.Match(i) || h_CPlusPlus_Matcher_6.Match(i) { } else if h_CPlusPlus_Matcher_0.Match(i) || h_CPlusPlus_Matcher_1.Match(i) || h_CPlusPlus_Matcher_2.Match(i) || h_CPlusPlus_Matcher_3.Match(i) || h_CPlusPlus_Matcher_4.Match(i) || h_CPlusPlus_Matcher_5.Match(i) || h_CPlusPlus_Matcher_6.Match(i) {
return "C++", true return []string{"C++"}
} }
return OtherLanguage, false return nil
}, },
".lsp": func(i []byte) (string, bool) { ".lsp": func(i []byte) []string {
if lsp_CommonLisp_Matcher_0.Match(i) { if lsp_CommonLisp_Matcher_0.Match(i) {
return "Common Lisp", true return []string{"Common Lisp"}
} else if lsp_NewLisp_Matcher_0.Match(i) { } else if lsp_NewLisp_Matcher_0.Match(i) {
return "NewLisp", true return []string{"NewLisp"}
} }
return OtherLanguage, false return nil
}, },
".lisp": func(i []byte) (string, bool) { ".lisp": func(i []byte) []string {
if lisp_CommonLisp_Matcher_0.Match(i) { if lisp_CommonLisp_Matcher_0.Match(i) {
return "Common Lisp", true return []string{"Common Lisp"}
} else if lisp_NewLisp_Matcher_0.Match(i) { } else if lisp_NewLisp_Matcher_0.Match(i) {
return "NewLisp", true return []string{"NewLisp"}
} }
return OtherLanguage, false return nil
}, },
".md": func(i []byte) (string, bool) { ".md": func(i []byte) []string {
if md_Markdown_Matcher_0.Match(i) || md_Markdown_Matcher_1.Match(i) { if md_Markdown_Matcher_0.Match(i) || md_Markdown_Matcher_1.Match(i) {
return "Markdown", true return []string{"Markdown"}
} else if md_GCCmachinedescription_Matcher_0.Match(i) { } else if md_GCCmachinedescription_Matcher_0.Match(i) {
return "GCC machine description", true return []string{"GCC machine description"}
} }
return "Markdown", true return []string{"Markdown"}
}, },
".ms": func(i []byte) (string, bool) { ".ms": func(i []byte) []string {
if ms_Groff_Matcher_0.Match(i) { if ms_Groff_Matcher_0.Match(i) {
return "Groff", true return []string{"Groff"}
} }
return "MAXScript", true return []string{"MAXScript"}
}, },
".mod": func(i []byte) (string, bool) { ".mod": func(i []byte) []string {
if mod_XML_Matcher_0.Match(i) { if mod_XML_Matcher_0.Match(i) {
return "XML", true return []string{"XML"}
} else if mod_ModulaDash2_Matcher_0.Match(i) || mod_ModulaDash2_Matcher_1.Match(i) { } else if mod_ModulaDash2_Matcher_0.Match(i) || mod_ModulaDash2_Matcher_1.Match(i) {
return "Modula-2", true return []string{"Modula-2"}
} }
return "Linux Kernel Module", false return []string{"Linux Kernel Module", "AMPL"}
}, },
".pro": func(i []byte) (string, bool) { ".pro": func(i []byte) []string {
if pro_Prolog_Matcher_0.Match(i) { if pro_Prolog_Matcher_0.Match(i) {
return "Prolog", true return []string{"Prolog"}
} else if pro_INI_Matcher_0.Match(i) { } else if pro_INI_Matcher_0.Match(i) {
return "INI", true return []string{"INI"}
} else if pro_QMake_Matcher_0.Match(i) && pro_QMake_Matcher_1.Match(i) { } else if pro_QMake_Matcher_0.Match(i) && pro_QMake_Matcher_1.Match(i) {
return "QMake", true return []string{"QMake"}
} else if pro_IDL_Matcher_0.Match(i) { } else if pro_IDL_Matcher_0.Match(i) {
return "IDL", true return []string{"IDL"}
} }
return OtherLanguage, false return nil
}, },
".rpy": func(i []byte) (string, bool) { ".rpy": func(i []byte) []string {
if rpy_Python_Matcher_0.Match(i) { if rpy_Python_Matcher_0.Match(i) {
return "Python", true return []string{"Python"}
} }
return "Ren'Py", true return []string{"Ren'Py"}
}, },
} }

2
internal/code-generator/generator/test_files/documentation.gold
View File

@ -2,7 +2,7 @@ package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND // THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7 // Extracted from github/linguist commit: 0123456789abcdef0123456789abcdef01234567
import "gopkg.in/toqueteos/substring.v1" import "gopkg.in/toqueteos/substring.v1"

8
internal/code-generator/generator/test_files/extensions.gold
View File

@ -2,7 +2,7 @@ package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND // THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7 // Extracted from github/linguist commit: 0123456789abcdef0123456789abcdef01234567
var languagesByExtension = map[string][]string{ var languagesByExtension = map[string][]string{
".abap": {"ABAP"}, ".abap": {"ABAP"},
@ -10,3 +10,9 @@ var languagesByExtension = map[string][]string{
".bsl": {"1C Enterprise"}, ".bsl": {"1C Enterprise"},
".os": {"1C Enterprise"}, ".os": {"1C Enterprise"},
} }
var extensionsByLanguage = map[string][]string{
"1C Enterprise": {".bsl", ".os"},
"ABAP": {".abap"},
"ABNF": {".abnf"},
}

12
internal/code-generator/generator/test_files/filenames.gold
View File

@ -2,11 +2,11 @@ package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND // THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7 // Extracted from github/linguist commit: 0123456789abcdef0123456789abcdef01234567
var languagesByFilename = map[string]string{ var languagesByFilename = map[string][]string{
"APKBUILD": "Alpine Abuild", "APKBUILD": {"Alpine Abuild"},
"CMakeLists.txt": "CMake", "CMakeLists.txt": {"CMake"},
"Cakefile": "CoffeeScript", "Cakefile": {"CoffeeScript"},
"mix.lock": "Elixir", "mix.lock": {"Elixir"},
} }

74224
internal/code-generator/generator/test_files/frequencies.gold
View File

File diff suppressed because it is too large Load Diff

2
internal/code-generator/generator/test_files/interpreters.gold
View File

@ -2,7 +2,7 @@ package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND // THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7 // Extracted from github/linguist commit: 0123456789abcdef0123456789abcdef01234567
var languagesByInterpreter = map[string][]string{ var languagesByInterpreter = map[string][]string{
"bash": {"Shell"}, "bash": {"Shell"},

2
internal/code-generator/generator/test_files/types.gold
View File

@ -2,7 +2,7 @@ package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND // THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7 // Extracted from github/linguist commit: 0123456789abcdef0123456789abcdef01234567
var languagesType = map[string]Type{ var languagesType = map[string]Type{
"Scaml": Markup, "Scaml": Markup,

2
internal/code-generator/generator/test_files/vendor.gold
View File

@ -2,7 +2,7 @@ package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND // THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7 // Extracted from github/linguist commit: 0123456789abcdef0123456789abcdef01234567
import "gopkg.in/toqueteos/substring.v1" import "gopkg.in/toqueteos/substring.v1"

2
internal/code-generator/generator/types.go
View File

@ -15,7 +15,7 @@ var typeToTypeConst = map[string]string{
"prose": "Prose", "prose": "Prose",
} }
// Types reads from buf and builds type.go file from typeTmplPath. // Types reads from buf and builds source file from typeTmplPath.
func Types(data []byte, typeTmplPath, typeTmplName, commit string) ([]byte, error) { func Types(data []byte, typeTmplPath, typeTmplName, commit string) ([]byte, error) {
languages := make(map[string]*languageInfo) languages := make(map[string]*languageInfo)
if err := yaml.Unmarshal(data, &languages); err != nil { if err := yaml.Unmarshal(data, &languages); err != nil {

4
internal/code-generator/generator/vendor.go
View File

@ -2,13 +2,13 @@ package generator
import ( import (
"bytes" "bytes"
"html/template"
"io" "io"
"text/template"
yaml "gopkg.in/yaml.v2" yaml "gopkg.in/yaml.v2"
) )
// Vendor reads from buf and builds vendor_matchers.go file from vendorTmplPath. // Vendor reads from buf and builds source file from vendorTmplPath.
func Vendor(data []byte, vendorTmplPath, vendorTmplName, commit string) ([]byte, error) { func Vendor(data []byte, vendorTmplPath, vendorTmplName, commit string) ([]byte, error) {
var regexpList []string var regexpList []string
if err := yaml.Unmarshal(data, &regexpList); err != nil { if err := yaml.Unmarshal(data, &regexpList); err != nil {

26
internal/code-generator/main.go
View File

@ -11,8 +11,8 @@ const (
// languages info file // languages info file
languagesYAML = ".linguist/lib/linguist/languages.yml" languagesYAML = ".linguist/lib/linguist/languages.yml"
// extensions_map.go generation // extension.go generation
extensionsFile = "extensions_map.go" extensionsFile = "extension.go"
extensionsTmplPath = "internal/code-generator/assets/extensions.go.tmpl" extensionsTmplPath = "internal/code-generator/assets/extensions.go.tmpl"
extensionsTmpl = "extensions.go.tmpl" extensionsTmpl = "extensions.go.tmpl"
@ -22,35 +22,35 @@ const (
contentTmplPath = "internal/code-generator/assets/content.go.tmpl" contentTmplPath = "internal/code-generator/assets/content.go.tmpl"
contentTmpl = "content.go.tmpl" contentTmpl = "content.go.tmpl"
// vendor_matchers.go generation // vendor.go generation
vendorYAML = ".linguist/lib/linguist/vendor.yml" vendorYAML = ".linguist/lib/linguist/vendor.yml"
vendorFile = "vendor_matchers.go" vendorFile = "vendor.go"
vendorTmplPath = "internal/code-generator/assets/vendor.go.tmpl" vendorTmplPath = "internal/code-generator/assets/vendor.go.tmpl"
vendorTmpl = "vendor.go.tmpl" vendorTmpl = "vendor.go.tmpl"
// documentation_matchers.go generation // documentation.go generation
documentationYAML = ".linguist/lib/linguist/documentation.yml" documentationYAML = ".linguist/lib/linguist/documentation.yml"
documentationFile = "documentation_matchers.go" documentationFile = "documentation.go"
documentationTmplPath = "internal/code-generator/assets/documentation.go.tmpl" documentationTmplPath = "internal/code-generator/assets/documentation.go.tmpl"
documentationTmpl = "documentation.go.tmpl" documentationTmpl = "documentation.go.tmpl"
// type.go generation // type.go generation
typeFile = "types_map.go" typeFile = "type.go"
typeTmplPath = "internal/code-generator/assets/types.go.tmpl" typeTmplPath = "internal/code-generator/assets/types.go.tmpl"
typeTmpl = "types.go.tmpl" typeTmpl = "types.go.tmpl"
// interpreters_map.go generation // interpreter.go generation
interpretersFile = "interpreters_map.go" interpretersFile = "interpreter.go"
interpretersTmplPath = "internal/code-generator/assets/interpreters.go.tmpl" interpretersTmplPath = "internal/code-generator/assets/interpreters.go.tmpl"
interpretersTmpl = "interpreters.go.tmpl" interpretersTmpl = "interpreters.go.tmpl"
// filenames_map.go generation // filename.go generation
filenamesFile = "filenames_map.go" filenamesFile = "filename.go"
filenamesTmplPath = "internal/code-generator/assets/filenames.go.tmpl" filenamesTmplPath = "internal/code-generator/assets/filenames.go.tmpl"
filenamesTmpl = "filenames.go.tmpl" filenamesTmpl = "filenames.go.tmpl"
// aliases_map.go generation // alias.go generation
aliasesFile = "aliases_map.go" aliasesFile = "alias.go"
aliasesTmplPath = "internal/code-generator/assets/aliases.go.tmpl" aliasesTmplPath = "internal/code-generator/assets/aliases.go.tmpl"
aliasesTmpl = "aliases.go.tmpl" aliasesTmpl = "aliases.go.tmpl"

6
internal/tokenizer/tokenize.go
View File

@ -5,7 +5,13 @@ import (
"regexp" "regexp"
) )
const byteLimit = 100000
func Tokenize(content []byte) []string { func Tokenize(content []byte) []string {
if len(content) > byteLimit {
content = content[:byteLimit]
}
tokens := make([][]byte, 0, 50) tokens := make([][]byte, 0, 50)
for _, extract := range extractTokens { for _, extract := range extractTokens {
var extractedTokens [][]byte var extractedTokens [][]byte

2
interpreters_map.go → interpreter.go
View File

@ -2,7 +2,7 @@ package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND // THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: 60f864a138650dd17fafc94814be9ee2d3aaef8c // Extracted from github/linguist commit: b6460f8ed6b249281ada099ca28bd8f1230b8892
var languagesByInterpreter = map[string][]string{ var languagesByInterpreter = map[string][]string{
"Rscript": {"R"}, "Rscript": {"R"},

105
modeline.go
View File

@ -5,40 +5,60 @@ import (
"regexp" "regexp"
) )
// GetLanguageByModeline returns the language of the given content looking for the modeline, const (
// and safe to indicate the sureness of returned language. searchScope = 5
func GetLanguageByModeline(content []byte) (lang string, safe bool) { )
// GetLanguagesByModeline returns a slice of possible languages for the given content, filename will be ignored.
// It accomplish the signature to be a Strategy type.
func GetLanguagesByModeline(filename string, content []byte) []string {
headFoot := getHeaderAndFooter(content) headFoot := getHeaderAndFooter(content)
var languages []string
for _, getLang := range modelinesFunc { for _, getLang := range modelinesFunc {
lang, safe = getLang(headFoot) languages = getLang("", headFoot)
if safe { if len(languages) > 0 {
break break
} }
} }
return return languages
} }
func getHeaderAndFooter(content []byte) []byte { func getHeaderAndFooter(content []byte) []byte {
const ( if bytes.Count(content, []byte("\n")) < 2*searchScope {
searchScope = 5
eol = "\n"
)
if bytes.Count(content, []byte(eol)) < 2*searchScope {
return content return content
} }
splitted := bytes.Split(content, []byte(eol)) header := headScope(content, searchScope)
header := splitted[:searchScope] footer := footScope(content, searchScope)
footer := splitted[len(splitted)-searchScope:] headerAndFooter := make([]byte, 0, len(content[:header])+len(content[footer:]))
headerAndFooter := append(header, footer...) headerAndFooter = append(headerAndFooter, content[:header]...)
return bytes.Join(headerAndFooter, []byte(eol)) headerAndFooter = append(headerAndFooter, content[footer:]...)
return headerAndFooter
} }
var modelinesFunc = []func(content []byte) (string, bool){ func headScope(content []byte, scope int) (index int) {
GetLanguageByEmacsModeline, for i := 0; i < scope; i++ {
GetLanguageByVimModeline, eol := bytes.IndexAny(content, "\n")
content = content[eol+1:]
index += eol
}
return index + scope - 1
}
func footScope(content []byte, scope int) (index int) {
for i := 0; i < scope; i++ {
index = bytes.LastIndexAny(content, "\n")
content = content[:index]
}
return index + 1
}
var modelinesFunc = []func(filename string, content []byte) []string{
GetLanguagesByEmacsModeline,
GetLanguagesByVimModeline,
} }
var ( var (
@ -51,9 +71,20 @@ var (
// GetLanguageByEmacsModeline detecs if the content has a emacs modeline and try to get a // GetLanguageByEmacsModeline detecs if the content has a emacs modeline and try to get a
// language basing on alias. If couldn't retrieve a valid language, it returns OtherLanguage and false. // language basing on alias. If couldn't retrieve a valid language, it returns OtherLanguage and false.
func GetLanguageByEmacsModeline(content []byte) (string, bool) { func GetLanguageByEmacsModeline(content []byte) (string, bool) {
languages := GetLanguagesByEmacsModeline("", content)
if len(languages) == 0 {
return OtherLanguage, false
}
return languages[0], true
}
// GetLanguagesByEmacsModeline returns a slice of possible languages for the given content, filename will be ignored.
// It accomplish the signature to be a Strategy type.
func GetLanguagesByEmacsModeline(filename string, content []byte) []string {
matched := reEmacsModeline.FindAllSubmatch(content, -1) matched := reEmacsModeline.FindAllSubmatch(content, -1)
if matched == nil { if matched == nil {
return OtherLanguage, false return nil
} }
// only take the last matched line, discard previous lines // only take the last matched line, discard previous lines
@ -66,22 +97,38 @@ func GetLanguageByEmacsModeline(content []byte) (string, bool) {
alias = string(lastLineMatched) alias = string(lastLineMatched)
} }
return GetLanguageByAlias(alias) language, ok := GetLanguageByAlias(alias)
if !ok {
return nil
}
return []string{language}
} }
// GetLanguageByVimModeline detecs if the content has a vim modeline and try to get a // GetLanguageByVimModeline detecs if the content has a vim modeline and try to get a
// language basing on alias. If couldn't retrieve a valid language, it returns OtherLanguage and false. // language basing on alias. If couldn't retrieve a valid language, it returns OtherLanguage and false.
func GetLanguageByVimModeline(content []byte) (string, bool) { func GetLanguageByVimModeline(content []byte) (string, bool) {
languages := GetLanguagesByVimModeline("", content)
if len(languages) == 0 {
return OtherLanguage, false
}
return languages[0], true
}
// GetLanguagesByVimModeline returns a slice of possible languages for the given content, filename will be ignored.
// It accomplish the signature to be a Strategy type.
func GetLanguagesByVimModeline(filename string, content []byte) []string {
matched := reVimModeline.FindAllSubmatch(content, -1) matched := reVimModeline.FindAllSubmatch(content, -1)
if matched == nil { if matched == nil {
return OtherLanguage, false return nil
} }
// only take the last matched line, discard previous lines // only take the last matched line, discard previous lines
lastLineMatched := matched[len(matched)-1][1] lastLineMatched := matched[len(matched)-1][1]
matchedAlias := reVimLang.FindAllSubmatch(lastLineMatched, -1) matchedAlias := reVimLang.FindAllSubmatch(lastLineMatched, -1)
if matchedAlias == nil { if matchedAlias == nil {
return OtherLanguage, false return nil
} }
alias := string(matchedAlias[0][1]) alias := string(matchedAlias[0][1])
@ -92,11 +139,15 @@ func GetLanguageByVimModeline(content []byte) (string, bool) {
for _, match := range matchedAlias { for _, match := range matchedAlias {
otherAlias := string(match[1]) otherAlias := string(match[1])
if otherAlias != alias { if otherAlias != alias {
alias = OtherLanguage return nil
break
} }
} }
} }
return GetLanguageByAlias(alias) language, ok := GetLanguageByAlias(alias)
if !ok {
return nil
}
return []string{language}
} }

87
modeline_test.go
View File

@ -1,87 +0,0 @@
package slinguist
import (
"io/ioutil"
"path/filepath"
. "gopkg.in/check.v1"
)
const (
modelinesDir = ".linguist/test/fixtures/Data/Modelines"
samplesDir = ".linguist/samples"
)
func (s *TSuite) TestGetLanguageByModeline(c *C) {
linguistTests := []struct {
filename string
expectedLang string
expectedSafe bool
}{
// Emacs
{filename: filepath.Join(modelinesDir, "example_smalltalk.md"), expectedLang: "Smalltalk", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "fundamentalEmacs.c"), expectedLang: "Text", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "iamphp.inc"), expectedLang: "PHP", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "seeplusplusEmacs1"), expectedLang: "C++", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "seeplusplusEmacs2"), expectedLang: "C++", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "seeplusplusEmacs3"), expectedLang: "C++", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "seeplusplusEmacs4"), expectedLang: "C++", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "seeplusplusEmacs5"), expectedLang: "C++", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "seeplusplusEmacs6"), expectedLang: "C++", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "seeplusplusEmacs7"), expectedLang: "C++", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "seeplusplusEmacs9"), expectedLang: "C++", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "seeplusplusEmacs10"), expectedLang: "C++", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "seeplusplusEmacs11"), expectedLang: "C++", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "seeplusplusEmacs12"), expectedLang: "C++", expectedSafe: true},
// Vim
{filename: filepath.Join(modelinesDir, "seeplusplus"), expectedLang: "C++", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "iamjs.pl"), expectedLang: "JavaScript", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "iamjs2.pl"), expectedLang: "JavaScript", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "not_perl.pl"), expectedLang: "Prolog", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "ruby"), expectedLang: "Ruby", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "ruby2"), expectedLang: "Ruby", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "ruby3"), expectedLang: "Ruby", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "ruby4"), expectedLang: "Ruby", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "ruby5"), expectedLang: "Ruby", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "ruby6"), expectedLang: "Ruby", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "ruby7"), expectedLang: "Ruby", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "ruby8"), expectedLang: "Ruby", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "ruby9"), expectedLang: "Ruby", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "ruby10"), expectedLang: "Ruby", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "ruby11"), expectedLang: "Ruby", expectedSafe: true},
{filename: filepath.Join(modelinesDir, "ruby12"), expectedLang: "Ruby", expectedSafe: true},
{filename: filepath.Join(samplesDir, "C/main.c"), expectedLang: OtherLanguage, expectedSafe: false},
}
for _, test := range linguistTests {
content, err := ioutil.ReadFile(test.filename)
c.Assert(err, Equals, nil)
lang, safe := GetLanguageByModeline(content)
c.Assert(lang, Equals, test.expectedLang)
c.Assert(safe, Equals, test.expectedSafe)
}
const (
wrongVim = `# vim: set syntax=ruby ft =python filetype=perl :`
rightVim = `/* vim: set syntax=python ft =python filetype=python */`
noLangVim = `/* vim: set shiftwidth=4 softtabstop=0 cindent cinoptions={1s: */`
)
tests := []struct {
content []byte
expectedLang string
expectedSafe bool
}{
{content: []byte(wrongVim), expectedLang: OtherLanguage, expectedSafe: false},
{content: []byte(rightVim), expectedLang: "Python", expectedSafe: true},
{content: []byte(noLangVim), expectedLang: OtherLanguage, expectedSafe: false},
}
for _, test := range tests {
lang, safe := GetLanguageByModeline(test.content)
c.Assert(lang, Equals, test.expectedLang)
c.Assert(safe, Equals, test.expectedSafe)
}
}

14
shebang.go
View File

@ -14,17 +14,11 @@ var (
pythonVersion = regexp.MustCompile(`python\d\.\d+`) pythonVersion = regexp.MustCompile(`python\d\.\d+`)
) )
// GetLanguageByShebang returns the language of the given content looking for the shebang line, // GetLanguagesByShebang returns a slice of possible languages for the given content, filename will be ignored.
// and safe to indicate the sureness of returned language. // It accomplish the signature to be a Strategy type.
func GetLanguageByShebang(content []byte) (lang string, safe bool) { func GetLanguagesByShebang(filename string, content []byte) (languages []string) {
interpreter := getInterpreter(content) interpreter := getInterpreter(content)
lang = OtherLanguage return languagesByInterpreter[interpreter]
if langs, ok := languagesByInterpreter[interpreter]; ok {
lang = langs[0]
safe = len(langs) == 1
}
return
} }
func getInterpreter(data []byte) (interpreter string) { func getInterpreter(data []byte) (interpreter string) {

60
shebang_test.go
View File

@ -1,60 +0,0 @@
package slinguist
import . "gopkg.in/check.v1"
const (
multilineExecHack = `#!/bin/sh
# Next line is comment in Tcl, but not in sh... \
exec tclsh "$0" ${1+"$@"}`
multilineNoExecHack = `#!/bin/sh
#<<<#
echo "A shell script in a zkl program ($0)"
echo "Now run zkl <this file> with Hello World as args"
zkl $0 Hello World!
exit
#<<<#
println("The shell script says ",vm.arglist.concat(" "));`
)
func (s *TSuite) TestGetLanguageByShebang(c *C) {
lang, safe := GetLanguageByShebang([]byte(`#!/unknown/interpreter`))
c.Assert(lang, Equals, OtherLanguage)
c.Assert(safe, Equals, false)
lang, safe = GetLanguageByShebang([]byte(`no shebang`))
c.Assert(lang, Equals, OtherLanguage)
c.Assert(safe, Equals, false)
lang, safe = GetLanguageByShebang([]byte(`#!/usr/bin/env`))
c.Assert(lang, Equals, OtherLanguage)
c.Assert(safe, Equals, false)
lang, safe = GetLanguageByShebang([]byte(`#!/usr/bin/python -tt`))
c.Assert(lang, Equals, "Python")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByShebang([]byte(`#!/usr/bin/env python2.6`))
c.Assert(lang, Equals, "Python")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByShebang([]byte(`#!/usr/bin/env perl`))
c.Assert(lang, Equals, "Perl")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByShebang([]byte(`#! /bin/sh`))
c.Assert(lang, Equals, "Shell")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByShebang([]byte(`#!bash`))
c.Assert(lang, Equals, "Shell")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByShebang([]byte(multilineExecHack))
c.Assert(lang, Equals, "Tcl")
c.Assert(safe, Equals, true)
lang, safe = GetLanguageByShebang([]byte(multilineNoExecHack))
c.Assert(lang, Equals, "Shell")
c.Assert(safe, Equals, true)
}

473
type.go
View File

@ -1,22 +1,459 @@
package slinguist package slinguist
type Type int // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: b6460f8ed6b249281ada099ca28bd8f1230b8892
const ( var languagesType = map[string]Type{
// Language's type. Either data, programming, markup, prose, or unknown. "1C Enterprise": Programming,
Unknown Type = iota "ABAP": Programming,
Data "ABNF": Data,
Programming "AGS Script": Programming,
Markup "AMPL": Programming,
Prose "ANTLR": Programming,
) "API Blueprint": Markup,
"APL": Programming,
// GetLanguageType returns the given language's type. "ASN.1": Data,
func GetLanguageType(language string) (langType Type) { "ASP": Programming,
langType, ok := languagesType[language] "ATS": Programming,
if !ok { "ActionScript": Programming,
langType = Unknown "Ada": Programming,
} "Agda": Programming,
"Alloy": Programming,
return langType "Alpine Abuild": Programming,
"Ant Build System": Data,
"ApacheConf": Markup,
"Apex": Programming,
"Apollo Guidance Computer": Programming,
"AppleScript": Programming,
"Arc": Programming,
"Arduino": Programming,
"AsciiDoc": Prose,
"AspectJ": Programming,
"Assembly": Programming,
"Augeas": Programming,
"AutoHotkey": Programming,
"AutoIt": Programming,
"Awk": Programming,
"Batchfile": Programming,
"Befunge": Programming,
"Bison": Programming,
"BitBake": Programming,
"Blade": Markup,
"BlitzBasic": Programming,
"BlitzMax": Programming,
"Bluespec": Programming,
"Boo": Programming,
"Brainfuck": Programming,
"Brightscript": Programming,
"Bro": Programming,
"C": Programming,
"C#": Programming,
"C++": Programming,
"C-ObjDump": Data,
"C2hs Haskell": Programming,
"CLIPS": Programming,
"CMake": Programming,
"COBOL": Programming,
"COLLADA": Data,
"CSON": Data,
"CSS": Markup,
"CSV": Data,
"CWeb": Programming,
"Cap'n Proto": Programming,
"CartoCSS": Programming,
"Ceylon": Programming,
"Chapel": Programming,
"Charity": Programming,
"ChucK": Programming,
"Cirru": Programming,
"Clarion": Programming,
"Clean": Programming,
"Click": Programming,
"Clojure": Programming,
"Closure Templates": Markup,
"CoffeeScript": Programming,
"ColdFusion": Programming,
"ColdFusion CFC": Programming,
"Common Lisp": Programming,
"Component Pascal": Programming,
"Cool": Programming,
"Coq": Programming,
"Cpp-ObjDump": Data,
"Creole": Prose,
"Crystal": Programming,
"Csound": Programming,
"Csound Document": Programming,
"Csound Score": Programming,
"Cuda": Programming,
"Cycript": Programming,
"Cython": Programming,
"D": Programming,
"D-ObjDump": Data,
"DIGITAL Command Language": Programming,
"DM": Programming,
"DNS Zone": Data,
"DTrace": Programming,
"Darcs Patch": Data,
"Dart": Programming,
"Diff": Data,
"Dockerfile": Data,
"Dogescript": Programming,
"Dylan": Programming,
"E": Programming,
"EBNF": Data,
"ECL": Programming,
"ECLiPSe": Programming,
"EJS": Markup,
"EQ": Programming,
"Eagle": Markup,
"Ecere Projects": Data,
"Eiffel": Programming,
"Elixir": Programming,
"Elm": Programming,
"Emacs Lisp": Programming,
"EmberScript": Programming,
"Erlang": Programming,
"F#": Programming,
"FLUX": Programming,
"Factor": Programming,
"Fancy": Programming,
"Fantom": Programming,
"Filebench WML": Programming,
"Filterscript": Programming,
"Formatted": Data,
"Forth": Programming,
"Fortran": Programming,
"FreeMarker": Programming,
"Frege": Programming,
"G-code": Data,
"GAMS": Programming,
"GAP": Programming,
"GCC Machine Description": Programming,
"GDB": Programming,
"GDScript": Programming,
"GLSL": Programming,
"GN": Data,
"Game Maker Language": Programming,
"Genie": Programming,
"Genshi": Programming,
"Gentoo Ebuild": Programming,
"Gentoo Eclass": Programming,
"Gettext Catalog": Prose,
"Gherkin": Programming,
"Glyph": Programming,
"Gnuplot": Programming,
"Go": Programming,
"Golo": Programming,
"Gosu": Programming,
"Grace": Programming,
"Gradle": Data,
"Grammatical Framework": Programming,
"Graph Modeling Language": Data,
"GraphQL": Data,
"Graphviz (DOT)": Data,
"Groovy": Programming,
"Groovy Server Pages": Programming,
"HCL": Programming,
"HLSL": Programming,
"HTML": Markup,
"HTML+Django": Markup,
"HTML+ECR": Markup,
"HTML+EEX": Markup,
"HTML+ERB": Markup,
"HTML+PHP": Markup,
"HTTP": Data,
"Hack": Programming,
"Haml": Markup,
"Handlebars": Markup,
"Harbour": Programming,
"Haskell": Programming,
"Haxe": Programming,
"Hy": Programming,
"HyPhy": Programming,
"IDL": Programming,
"IGOR Pro": Programming,
"INI": Data,
"IRC log": Data,
"Idris": Programming,
"Inform 7": Programming,
"Inno Setup": Programming,
"Io": Programming,
"Ioke": Programming,
"Isabelle": Programming,
"Isabelle ROOT": Programming,
"J": Programming,
"JFlex": Programming,
"JSON": Data,
"JSON5": Data,
"JSONLD": Data,
"JSONiq": Programming,
"JSX": Programming,
"Jasmin": Programming,
"Java": Programming,
"Java Server Pages": Programming,
"JavaScript": Programming,
"Jison": Programming,
"Jison Lex": Programming,
"Jolie": Programming,
"Julia": Programming,
"Jupyter Notebook": Markup,
"KRL": Programming,
"KiCad": Programming,
"Kit": Markup,
"Kotlin": Programming,
"LFE": Programming,
"LLVM": Programming,
"LOLCODE": Programming,
"LSL": Programming,
"LabVIEW": Programming,
"Lasso": Programming,
"Latte": Markup,
"Lean": Programming,
"Less": Markup,
"Lex": Programming,
"LilyPond": Programming,
"Limbo": Programming,
"Linker Script": Data,
"Linux Kernel Module": Data,
"Liquid": Markup,
"Literate Agda": Programming,
"Literate CoffeeScript": Programming,
"Literate Haskell": Programming,
"LiveScript": Programming,
"Logos": Programming,
"Logtalk": Programming,
"LookML": Programming,
"LoomScript": Programming,
"Lua": Programming,
"M": Programming,
"M4": Programming,
"M4Sugar": Programming,
"MAXScript": Programming,
"MQL4": Programming,
"MQL5": Programming,
"MTML": Markup,
"MUF": Programming,
"Makefile": Programming,
"Mako": Programming,
"Markdown": Prose,
"Marko": Markup,
"Mask": Markup,
"Mathematica": Programming,
"Matlab": Programming,
"Maven POM": Data,
"Max": Programming,
"MediaWiki": Prose,
"Mercury": Programming,
"Meson": Programming,
"Metal": Programming,
"MiniD": Programming,
"Mirah": Programming,
"Modelica": Programming,
"Modula-2": Programming,
"Module Management System": Programming,
"Monkey": Programming,
"Moocode": Programming,
"MoonScript": Programming,
"Myghty": Programming,
"NCL": Programming,
"NL": Data,
"NSIS": Programming,
"Nemerle": Programming,
"NetLinx": Programming,
"NetLinx+ERB": Programming,
"NetLogo": Programming,
"NewLisp": Programming,
"Nginx": Markup,
"Nim": Programming,
"Ninja": Data,
"Nit": Programming,
"Nix": Programming,
"Nu": Programming,
"NumPy": Programming,
"OCaml": Programming,
"ObjDump": Data,
"Objective-C": Programming,
"Objective-C++": Programming,
"Objective-J": Programming,
"Omgrofl": Programming,
"Opa": Programming,
"Opal": Programming,
"OpenCL": Programming,
"OpenEdge ABL": Programming,
"OpenRC runscript": Programming,
"OpenSCAD": Programming,
"OpenType Feature File": Data,
"Org": Prose,
"Ox": Programming,
"Oxygene": Programming,
"Oz": Programming,
"P4": Programming,
"PAWN": Programming,
"PHP": Programming,
"PLSQL": Programming,
"PLpgSQL": Programming,
"POV-Ray SDL": Programming,
"Pan": Programming,
"Papyrus": Programming,
"Parrot": Programming,
"Parrot Assembly": Programming,
"Parrot Internal Representation": Programming,
"Pascal": Programming,
"Pep8": Programming,
"Perl": Programming,
"Perl6": Programming,
"Pic": Markup,
"Pickle": Data,
"PicoLisp": Programming,
"PigLatin": Programming,
"Pike": Programming,
"Pod": Prose,
"PogoScript": Programming,
"Pony": Programming,
"PostScript": Markup,
"PowerBuilder": Programming,
"PowerShell": Programming,
"Processing": Programming,
"Prolog": Programming,
"Propeller Spin": Programming,
"Protocol Buffer": Markup,
"Public Key": Data,
"Pug": Markup,
"Puppet": Programming,
"Pure Data": Programming,
"PureBasic": Programming,
"PureScript": Programming,
"Python": Programming,
"Python console": Programming,
"Python traceback": Data,
"QML": Programming,
"QMake": Programming,
"R": Programming,
"RAML": Markup,
"RDoc": Prose,
"REALbasic": Programming,
"REXX": Programming,
"RHTML": Markup,
"RMarkdown": Prose,
"RPM Spec": Data,
"RUNOFF": Markup,
"Racket": Programming,
"Ragel": Programming,
"Rascal": Programming,
"Raw token data": Data,
"Reason": Programming,
"Rebol": Programming,
"Red": Programming,
"Redcode": Programming,
"Regular Expression": Data,
"Ren'Py": Programming,
"RenderScript": Programming,
"RobotFramework": Programming,
"Roff": Markup,
"Rouge": Programming,
"Ruby": Programming,
"Rust": Programming,
"SAS": Programming,
"SCSS": Markup,
"SMT": Programming,
"SPARQL": Data,
"SQF": Programming,
"SQL": Data,
"SQLPL": Programming,
"SRecode Template": Markup,
"STON": Data,
"SVG": Data,
"Sage": Programming,
"SaltStack": Programming,
"Sass": Markup,
"Scala": Programming,
"Scaml": Markup,
"Scheme": Programming,
"Scilab": Programming,
"Self": Programming,
"ShaderLab": Programming,
"Shell": Programming,
"ShellSession": Programming,
"Shen": Programming,
"Slash": Programming,
"Slim": Markup,
"Smali": Programming,
"Smalltalk": Programming,
"Smarty": Programming,
"SourcePawn": Programming,
"Spline Font Database": Data,
"Squirrel": Programming,
"Stan": Programming,
"Standard ML": Programming,
"Stata": Programming,
"Stylus": Markup,
"SubRip Text": Data,
"Sublime Text Config": Data,
"SuperCollider": Programming,
"Swift": Programming,
"SystemVerilog": Programming,
"TI Program": Programming,
"TLA": Programming,
"TOML": Data,
"TXL": Programming,
"Tcl": Programming,
"Tcsh": Programming,
"TeX": Markup,
"Tea": Markup,
"Terra": Programming,
"Text": Prose,
"Textile": Prose,
"Thrift": Programming,
"Turing": Programming,
"Turtle": Data,
"Twig": Markup,
"Type Language": Data,
"TypeScript": Programming,
"Unified Parallel C": Programming,
"Unity3D Asset": Data,
"Unix Assembly": Programming,
"Uno": Programming,
"UnrealScript": Programming,
"UrWeb": Programming,
"VCL": Programming,
"VHDL": Programming,
"Vala": Programming,
"Verilog": Programming,
"Vim script": Programming,
"Visual Basic": Programming,
"Volt": Programming,
"Vue": Markup,
"Wavefront Material": Data,
"Wavefront Object": Data,
"Web Ontology Language": Markup,
"WebAssembly": Programming,
"WebIDL": Programming,
"World of Warcraft Addon Data": Data,
"X10": Programming,
"XC": Programming,
"XCompose": Data,
"XML": Data,
"XPages": Programming,
"XProc": Programming,
"XQuery": Programming,
"XS": Programming,
"XSLT": Programming,
"Xojo": Programming,
"Xtend": Programming,
"YAML": Data,
"YANG": Data,
"Yacc": Programming,
"Zephir": Programming,
"Zimpl": Programming,
"desktop": Data,
"eC": Programming,
"edn": Data,
"fish": Programming,
"mupad": Programming,
"nesC": Programming,
"ooc": Programming,
"reStructuredText": Prose,
"wisp": Programming,
"xBase": Programming,
} }

35
type_test.go
View File

@ -1,35 +0,0 @@
package slinguist
import . "gopkg.in/check.v1"
func (s *TSuite) TestGetLanguageType(c *C) {
langType := GetLanguageType("BestLanguageEver")
c.Assert(langType, Equals, Unknown)
langType = GetLanguageType("JSON")
c.Assert(langType, Equals, Data)
langType = GetLanguageType("COLLADA")
c.Assert(langType, Equals, Data)
langType = GetLanguageType("Go")
c.Assert(langType, Equals, Programming)
langType = GetLanguageType("Brainfuck")
c.Assert(langType, Equals, Programming)
langType = GetLanguageType("HTML")
c.Assert(langType, Equals, Markup)
langType = GetLanguageType("Sass")
c.Assert(langType, Equals, Markup)
langType = GetLanguageType("AsciiDoc")
c.Assert(langType, Equals, Prose)
langType = GetLanguageType("Textile")
c.Assert(langType, Equals, Prose)
langType = GetLanguageType("Whatever")
c.Assert(langType, Equals, Unknown)
}

458
types_map.go
View File

@ -1,458 +0,0 @@
package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: 60f864a138650dd17fafc94814be9ee2d3aaef8c
var languagesType = map[string]Type{
"1C Enterprise": Programming,
"ABAP": Programming,
"ABNF": Data,
"AGS Script": Programming,
"AMPL": Programming,
"ANTLR": Programming,
"API Blueprint": Markup,
"APL": Programming,
"ASN.1": Data,
"ASP": Programming,
"ATS": Programming,
"ActionScript": Programming,
"Ada": Programming,
"Agda": Programming,
"Alloy": Programming,
"Alpine Abuild": Programming,
"Ant Build System": Data,
"ApacheConf": Markup,
"Apex": Programming,
"Apollo Guidance Computer": Programming,
"AppleScript": Programming,
"Arc": Programming,
"Arduino": Programming,
"AsciiDoc": Prose,
"AspectJ": Programming,
"Assembly": Programming,
"Augeas": Programming,
"AutoHotkey": Programming,
"AutoIt": Programming,
"Awk": Programming,
"Batchfile": Programming,
"Befunge": Programming,
"Bison": Programming,
"BitBake": Programming,
"Blade": Markup,
"BlitzBasic": Programming,
"BlitzMax": Programming,
"Bluespec": Programming,
"Boo": Programming,
"Brainfuck": Programming,
"Brightscript": Programming,
"Bro": Programming,
"C": Programming,
"C#": Programming,
"C++": Programming,
"C-ObjDump": Data,
"C2hs Haskell": Programming,
"CLIPS": Programming,
"CMake": Programming,
"COBOL": Programming,
"COLLADA": Data,
"CSON": Data,
"CSS": Markup,
"CSV": Data,
"CWeb": Programming,
"Cap'n Proto": Programming,
"CartoCSS": Programming,
"Ceylon": Programming,
"Chapel": Programming,
"Charity": Programming,
"ChucK": Programming,
"Cirru": Programming,
"Clarion": Programming,
"Clean": Programming,
"Click": Programming,
"Clojure": Programming,
"Closure Templates": Markup,
"CoffeeScript": Programming,
"ColdFusion": Programming,
"ColdFusion CFC": Programming,
"Common Lisp": Programming,
"Component Pascal": Programming,
"Cool": Programming,
"Coq": Programming,
"Cpp-ObjDump": Data,
"Creole": Prose,
"Crystal": Programming,
"Csound": Programming,
"Csound Document": Programming,
"Csound Score": Programming,
"Cuda": Programming,
"Cycript": Programming,
"Cython": Programming,
"D": Programming,
"D-ObjDump": Data,
"DIGITAL Command Language": Programming,
"DM": Programming,
"DNS Zone": Data,
"DTrace": Programming,
"Darcs Patch": Data,
"Dart": Programming,
"Diff": Data,
"Dockerfile": Data,
"Dogescript": Programming,
"Dylan": Programming,
"E": Programming,
"EBNF": Data,
"ECL": Programming,
"ECLiPSe": Programming,
"EJS": Markup,
"EQ": Programming,
"Eagle": Markup,
"Ecere Projects": Data,
"Eiffel": Programming,
"Elixir": Programming,
"Elm": Programming,
"Emacs Lisp": Programming,
"EmberScript": Programming,
"Erlang": Programming,
"F#": Programming,
"FLUX": Programming,
"Factor": Programming,
"Fancy": Programming,
"Fantom": Programming,
"Filebench WML": Programming,
"Filterscript": Programming,
"Formatted": Data,
"Forth": Programming,
"Fortran": Programming,
"FreeMarker": Programming,
"Frege": Programming,
"G-code": Data,
"GAMS": Programming,
"GAP": Programming,
"GCC Machine Description": Programming,
"GDB": Programming,
"GDScript": Programming,
"GLSL": Programming,
"GN": Data,
"Game Maker Language": Programming,
"Genie": Programming,
"Genshi": Programming,
"Gentoo Ebuild": Programming,
"Gentoo Eclass": Programming,
"Gettext Catalog": Prose,
"Gherkin": Programming,
"Glyph": Programming,
"Gnuplot": Programming,
"Go": Programming,
"Golo": Programming,
"Gosu": Programming,
"Grace": Programming,
"Gradle": Data,
"Grammatical Framework": Programming,
"Graph Modeling Language": Data,
"GraphQL": Data,
"Graphviz (DOT)": Data,
"Groovy": Programming,
"Groovy Server Pages": Programming,
"HCL": Programming,
"HLSL": Programming,
"HTML": Markup,
"HTML+Django": Markup,
"HTML+ECR": Markup,
"HTML+EEX": Markup,
"HTML+ERB": Markup,
"HTML+PHP": Markup,
"HTTP": Data,
"Hack": Programming,
"Haml": Markup,
"Handlebars": Markup,
"Harbour": Programming,
"Haskell": Programming,
"Haxe": Programming,
"Hy": Programming,
"HyPhy": Programming,
"IDL": Programming,
"IGOR Pro": Programming,
"INI": Data,
"IRC log": Data,
"Idris": Programming,
"Inform 7": Programming,
"Inno Setup": Programming,
"Io": Programming,
"Ioke": Programming,
"Isabelle": Programming,
"Isabelle ROOT": Programming,
"J": Programming,
"JFlex": Programming,
"JSON": Data,
"JSON5": Data,
"JSONLD": Data,
"JSONiq": Programming,
"JSX": Programming,
"Jasmin": Programming,
"Java": Programming,
"Java Server Pages": Programming,
"JavaScript": Programming,
"Jison": Programming,
"Jison Lex": Programming,
"Jolie": Programming,
"Julia": Programming,
"Jupyter Notebook": Markup,
"KRL": Programming,
"KiCad": Programming,
"Kit": Markup,
"Kotlin": Programming,
"LFE": Programming,
"LLVM": Programming,
"LOLCODE": Programming,
"LSL": Programming,
"LabVIEW": Programming,
"Lasso": Programming,
"Latte": Markup,
"Lean": Programming,
"Less": Markup,
"Lex": Programming,
"LilyPond": Programming,
"Limbo": Programming,
"Linker Script": Data,
"Linux Kernel Module": Data,
"Liquid": Markup,
"Literate Agda": Programming,
"Literate CoffeeScript": Programming,
"Literate Haskell": Programming,
"LiveScript": Programming,
"Logos": Programming,
"Logtalk": Programming,
"LookML": Programming,
"LoomScript": Programming,
"Lua": Programming,
"M": Programming,
"M4": Programming,
"M4Sugar": Programming,
"MAXScript": Programming,
"MQL4": Programming,
"MQL5": Programming,
"MTML": Markup,
"MUF": Programming,
"Makefile": Programming,
"Mako": Programming,
"Markdown": Prose,
"Marko": Markup,
"Mask": Markup,
"Mathematica": Programming,
"Matlab": Programming,
"Maven POM": Data,
"Max": Programming,
"MediaWiki": Prose,
"Mercury": Programming,
"Meson": Programming,
"Metal": Programming,
"MiniD": Programming,
"Mirah": Programming,
"Modelica": Programming,
"Modula-2": Programming,
"Module Management System": Programming,
"Monkey": Programming,
"Moocode": Programming,
"MoonScript": Programming,
"Myghty": Programming,
"NCL": Programming,
"NL": Data,
"NSIS": Programming,
"Nemerle": Programming,
"NetLinx": Programming,
"NetLinx+ERB": Programming,
"NetLogo": Programming,
"NewLisp": Programming,
"Nginx": Markup,
"Nim": Programming,
"Ninja": Data,
"Nit": Programming,
"Nix": Programming,
"Nu": Programming,
"NumPy": Programming,
"OCaml": Programming,
"ObjDump": Data,
"Objective-C": Programming,
"Objective-C++": Programming,
"Objective-J": Programming,
"Omgrofl": Programming,
"Opa": Programming,
"Opal": Programming,
"OpenCL": Programming,
"OpenEdge ABL": Programming,
"OpenRC runscript": Programming,
"OpenSCAD": Programming,
"OpenType Feature File": Data,
"Org": Prose,
"Ox": Programming,
"Oxygene": Programming,
"Oz": Programming,
"P4": Programming,
"PAWN": Programming,
"PHP": Programming,
"PLSQL": Programming,
"PLpgSQL": Programming,
"POV-Ray SDL": Programming,
"Pan": Programming,
"Papyrus": Programming,
"Parrot": Programming,
"Parrot Assembly": Programming,
"Parrot Internal Representation": Programming,
"Pascal": Programming,
"Pep8": Programming,
"Perl": Programming,
"Perl6": Programming,
"Pic": Markup,
"Pickle": Data,
"PicoLisp": Programming,
"PigLatin": Programming,
"Pike": Programming,
"Pod": Prose,
"PogoScript": Programming,
"Pony": Programming,
"PostScript": Markup,
"PowerBuilder": Programming,
"PowerShell": Programming,
"Processing": Programming,
"Prolog": Programming,
"Propeller Spin": Programming,
"Protocol Buffer": Markup,
"Public Key": Data,
"Pug": Markup,
"Puppet": Programming,
"Pure Data": Programming,
"PureBasic": Programming,
"PureScript": Programming,
"Python": Programming,
"Python console": Programming,
"Python traceback": Data,
"QML": Programming,
"QMake": Programming,
"R": Programming,
"RAML": Markup,
"RDoc": Prose,
"REALbasic": Programming,
"REXX": Programming,
"RHTML": Markup,
"RMarkdown": Prose,
"RPM Spec": Data,
"RUNOFF": Markup,
"Racket": Programming,
"Ragel": Programming,
"Rascal": Programming,
"Raw token data": Data,
"Reason": Programming,
"Rebol": Programming,
"Red": Programming,
"Redcode": Programming,
"Regular Expression": Data,
"Ren'Py": Programming,
"RenderScript": Programming,
"RobotFramework": Programming,
"Roff": Markup,
"Rouge": Programming,
"Ruby": Programming,
"Rust": Programming,
"SAS": Programming,
"SCSS": Markup,
"SMT": Programming,
"SPARQL": Data,
"SQF": Programming,
"SQL": Data,
"SQLPL": Programming,
"SRecode Template": Markup,
"STON": Data,
"SVG": Data,
"Sage": Programming,
"SaltStack": Programming,
"Sass": Markup,
"Scala": Programming,
"Scaml": Markup,
"Scheme": Programming,
"Scilab": Programming,
"Self": Programming,
"ShaderLab": Programming,
"Shell": Programming,
"ShellSession": Programming,
"Shen": Programming,
"Slash": Programming,
"Slim": Markup,
"Smali": Programming,
"Smalltalk": Programming,
"Smarty": Programming,
"SourcePawn": Programming,
"Spline Font Database": Data,
"Squirrel": Programming,
"Stan": Programming,
"Standard ML": Programming,
"Stata": Programming,
"Stylus": Markup,
"SubRip Text": Data,
"Sublime Text Config": Data,
"SuperCollider": Programming,
"Swift": Programming,
"SystemVerilog": Programming,
"TI Program": Programming,
"TLA": Programming,
"TOML": Data,
"TXL": Programming,
"Tcl": Programming,
"Tcsh": Programming,
"TeX": Markup,
"Tea": Markup,
"Terra": Programming,
"Text": Prose,
"Textile": Prose,
"Thrift": Programming,
"Turing": Programming,
"Turtle": Data,
"Twig": Markup,
"Type Language": Data,
"TypeScript": Programming,
"Unified Parallel C": Programming,
"Unity3D Asset": Data,
"Unix Assembly": Programming,
"Uno": Programming,
"UnrealScript": Programming,
"UrWeb": Programming,
"VCL": Programming,
"VHDL": Programming,
"Vala": Programming,
"Verilog": Programming,
"Vim script": Programming,
"Visual Basic": Programming,
"Volt": Programming,
"Vue": Markup,
"Wavefront Material": Data,
"Wavefront Object": Data,
"Web Ontology Language": Markup,
"WebIDL": Programming,
"World of Warcraft Addon Data": Data,
"X10": Programming,
"XC": Programming,
"XCompose": Data,
"XML": Data,
"XPages": Programming,
"XProc": Programming,
"XQuery": Programming,
"XS": Programming,
"XSLT": Programming,
"Xojo": Programming,
"Xtend": Programming,
"YAML": Data,
"YANG": Data,
"Yacc": Programming,
"Zephir": Programming,
"Zimpl": Programming,
"desktop": Data,
"eC": Programming,
"edn": Data,
"fish": Programming,
"mupad": Programming,
"nesC": Programming,
"ooc": Programming,
"reStructuredText": Prose,
"wisp": Programming,
"xBase": Programming,
}

36
utils.go
View File

@ -4,15 +4,32 @@ import (
"bytes" "bytes"
"path/filepath" "path/filepath"
"strings" "strings"
"gopkg.in/toqueteos/substring.v1"
) )
var (
auxiliaryLanguages = map[string]bool{
"Other": true, "XML": true, "YAML": true, "TOML": true, "INI": true,
"JSON": true, "TeX": true, "Public Key": true, "AsciiDoc": true,
"AGS Script": true, "VimL": true, "Diff": true, "CMake": true, "fish": true,
"Awk": true, "Graphviz (DOT)": true, "Markdown": true, "desktop": true,
"XSLT": true, "SQL": true, "RMarkdown": true, "IRC log": true,
"reStructuredText": true, "Twig": true, "CSS": true, "Batchfile": true,
"Text": true, "HTML+ERB": true, "HTML": true, "Gettext Catalog": true,
"Smarty": true, "Raw token data": true,
}
configurationLanguages = map[string]bool{
"XML": true, "JSON": true, "TOML": true, "YAML": true, "INI": true, "SQL": true,
}
)
// IsAuxiliaryLanguage returns whether or not lang is an auxiliary language.
func IsAuxiliaryLanguage(lang string) bool { func IsAuxiliaryLanguage(lang string) bool {
_, ok := auxiliaryLanguages[lang] _, ok := auxiliaryLanguages[lang]
return ok return ok
} }
// IsConfiguration returns whether or not path is using a configuration language.
func IsConfiguration(path string) bool { func IsConfiguration(path string) bool {
lang, _ := GetLanguageByExtension(path) lang, _ := GetLanguageByExtension(path)
_, is := configurationLanguages[lang] _, is := configurationLanguages[lang]
@ -20,20 +37,19 @@ func IsConfiguration(path string) bool {
return is return is
} }
// IsDotFile returns whether or not path has dot as a prefix.
func IsDotFile(path string) bool { func IsDotFile(path string) bool {
return strings.HasPrefix(filepath.Base(path), ".") return strings.HasPrefix(filepath.Base(path), ".")
} }
// IsVendor returns whether or not path is a vendor path.
func IsVendor(path string) bool { func IsVendor(path string) bool {
return findIndex(path, vendorMatchers) >= 0 return vendorMatchers.Match(path)
} }
// IsDocumentation returns whether or not path is a documentation path.
func IsDocumentation(path string) bool { func IsDocumentation(path string) bool {
return findIndex(path, documentationMatchers) >= 0 return documentationMatchers.Match(path)
}
func findIndex(path string, matchers substring.StringsMatcher) int {
return matchers.MatchIndex(path)
} }
const sniffLen = 8000 const sniffLen = 8000
@ -51,7 +67,3 @@ func IsBinary(data []byte) bool {
return true return true
} }
var configurationLanguages = map[string]bool{
"XML": true, "JSON": true, "TOML": true, "YAML": true, "INI": true, "SQL": true,
}

99
utils_test.go
View File

@ -2,42 +2,83 @@ package slinguist
import ( import (
"bytes" "bytes"
"fmt"
"testing"
. "gopkg.in/check.v1" "github.com/stretchr/testify/assert"
) )
func (s *TSuite) TestIsVendor(c *C) { func (s *SimpleLinguistTestSuite) TestIsVendor() {
c.Assert(IsVendor("foo/bar"), Equals, false) tests := []struct {
c.Assert(IsVendor("foo/vendor/foo"), Equals, true) name string
c.Assert(IsVendor(".sublime-project"), Equals, true) path string
c.Assert(IsVendor("leaflet.draw-src.js"), Equals, true) expected bool
c.Assert(IsVendor("foo/bar/MochiKit.js"), Equals, true) }{
c.Assert(IsVendor("foo/bar/dojo.js"), Equals, true) {name: "TestIsVendor_1", path: "foo/bar", expected: false},
c.Assert(IsVendor("foo/env/whatever"), Equals, true) {name: "TestIsVendor_2", path: "foo/vendor/foo", expected: true},
c.Assert(IsVendor("foo/.imageset/bar"), Equals, true) {name: "TestIsVendor_3", path: ".sublime-project", expected: true},
c.Assert(IsVendor("Vagrantfile"), Equals, true) {name: "TestIsVendor_4", path: "leaflet.draw-src.js", expected: true},
{name: "TestIsVendor_5", path: "foo/bar/MochiKit.js", expected: true},
{name: "TestIsVendor_6", path: "foo/bar/dojo.js", expected: true},
{name: "TestIsVendor_7", path: "foo/env/whatever", expected: true},
{name: "TestIsVendor_8", path: "foo/.imageset/bar", expected: true},
{name: "TestIsVendor_9", path: "Vagrantfile", expected: true},
} }
func (s *TSuite) TestIsDocumentation(c *C) { for _, test := range tests {
c.Assert(IsDocumentation("foo"), Equals, false) is := IsVendor(test.path)
c.Assert(IsDocumentation("README"), Equals, true) assert.Equal(s.T(), is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected))
}
} }
func (s *TSuite) TestIsConfiguration(c *C) { func (s *SimpleLinguistTestSuite) TestIsDocumentation() {
c.Assert(IsConfiguration("foo"), Equals, false) tests := []struct {
c.Assert(IsConfiguration("foo.ini"), Equals, true) name string
c.Assert(IsConfiguration("foo.json"), Equals, true) path string
expected bool
}{
{name: "TestIsDocumentation_1", path: "foo", expected: false},
{name: "TestIsDocumentation_2", path: "README", expected: true},
} }
func (s *TSuite) TestIsBinary(c *C) { for _, test := range tests {
c.Assert(IsBinary([]byte("foo")), Equals, false) is := IsDocumentation(test.path)
assert.Equal(s.T(), is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected))
}
}
binary := []byte{0} func (s *SimpleLinguistTestSuite) TestIsConfiguration() {
c.Assert(IsBinary(binary), Equals, true) tests := []struct {
name string
path string
expected bool
}{
{name: "TestIsConfiguration_1", path: "foo", expected: false},
{name: "TestIsConfiguration_2", path: "foo.ini", expected: true},
{name: "TestIsConfiguration_3", path: "foo.json", expected: true},
}
binary = bytes.Repeat([]byte{'o'}, 8000) for _, test := range tests {
binary = append(binary, byte(0)) is := IsConfiguration(test.path)
c.Assert(IsBinary(binary), Equals, false) assert.Equal(s.T(), is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected))
}
}
func (s *SimpleLinguistTestSuite) TestIsBinary() {
tests := []struct {
name string
data []byte
expected bool
}{
{name: "TestIsBinary_1", data: []byte("foo"), expected: false},
{name: "TestIsBinary_2", data: []byte{0}, expected: true},
{name: "TestIsBinary_3", data: bytes.Repeat([]byte{'o'}, 8000), expected: false},
}
for _, test := range tests {
is := IsBinary(test.data)
assert.Equal(s.T(), is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected))
}
} }
const ( const (
@ -45,14 +86,14 @@ const (
jsPath = "some/random/dir/file.js" jsPath = "some/random/dir/file.js"
) )
func (s *TSuite) BenchmarkVendor(c *C) { func BenchmarkVendor(b *testing.B) {
for i := 0; i < c.N; i++ { for i := 0; i < b.N; i++ {
_ = IsVendor(htmlPath) _ = IsVendor(htmlPath)
} }
} }
func (s *TSuite) BenchmarkVendorJS(c *C) { func BenchmarkVendorJS(b *testing.B) {
for i := 0; i < c.N; i++ { for i := 0; i < b.N; i++ {
_ = IsVendor(jsPath) _ = IsVendor(jsPath)
} }
} }

28
vendor_matchers.go → vendor.go
View File

@ -2,7 +2,7 @@ package slinguist
// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator // CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
// THIS FILE SHOULD NOT BE EDITED BY HAND // THIS FILE SHOULD NOT BE EDITED BY HAND
// Extracted from github/linguist commit: 60f864a138650dd17fafc94814be9ee2d3aaef8c // Extracted from github/linguist commit: b6460f8ed6b249281ada099ca28bd8f1230b8892
import "gopkg.in/toqueteos/substring.v1" import "gopkg.in/toqueteos/substring.v1"
@ -41,20 +41,20 @@ var vendorMatchers = substring.Or(
substring.Regexp(`3rd[-_]?party/`), substring.Regexp(`3rd[-_]?party/`),
substring.Regexp(`vendors?/`), substring.Regexp(`vendors?/`),
substring.Regexp(`extern(al)?/`), substring.Regexp(`extern(al)?/`),
substring.Regexp(`(^|/)[Vv]&#43;endor/`), substring.Regexp(`(^|/)[Vv]+endor/`),
substring.Regexp(`^debian/`), substring.Regexp(`^debian/`),
substring.Regexp(`run.n$`), substring.Regexp(`run.n$`),
substring.Regexp(`bootstrap-datepicker/`), substring.Regexp(`bootstrap-datepicker/`),
substring.Regexp(`(^|/)jquery([^.]*)\.js$`), substring.Regexp(`(^|/)jquery([^.]*)\.js$`),
substring.Regexp(`(^|/)jquery\-\d\.\d&#43;(\.\d&#43;)?\.js$`), substring.Regexp(`(^|/)jquery\-\d\.\d+(\.\d+)?\.js$`),
substring.Regexp(`(^|/)jquery\-ui(\-\d\.\d&#43;(\.\d&#43;)?)?(\.\w&#43;)?\.(js|css)$`), substring.Regexp(`(^|/)jquery\-ui(\-\d\.\d+(\.\d+)?)?(\.\w+)?\.(js|css)$`),
substring.Regexp(`(^|/)jquery\.(ui|effects)\.([^.]*)\.(js|css)$`), substring.Regexp(`(^|/)jquery\.(ui|effects)\.([^.]*)\.(js|css)$`),
substring.Regexp(`jquery.fn.gantt.js`), substring.Regexp(`jquery.fn.gantt.js`),
substring.Regexp(`jquery.fancybox.(js|css)`), substring.Regexp(`jquery.fancybox.(js|css)`),
substring.Regexp(`fuelux.js`), substring.Regexp(`fuelux.js`),
substring.Regexp(`(^|/)jquery\.fileupload(-\w&#43;)?\.js$`), substring.Regexp(`(^|/)jquery\.fileupload(-\w+)?\.js$`),
substring.Regexp(`(^|/)slick\.\w&#43;.js$`), substring.Regexp(`(^|/)slick\.\w+.js$`),
substring.Regexp(`(^|/)Leaflet\.Coordinates-\d&#43;\.\d&#43;\.\d&#43;\.src\.js$`), substring.Regexp(`(^|/)Leaflet\.Coordinates-\d+\.\d+\.\d+\.src\.js$`),
substring.Regexp(`leaflet.draw-src.js`), substring.Regexp(`leaflet.draw-src.js`),
substring.Regexp(`leaflet.draw.css`), substring.Regexp(`leaflet.draw.css`),
substring.Regexp(`Control.FullScreen.css`), substring.Regexp(`Control.FullScreen.css`),
@ -68,7 +68,7 @@ var vendorMatchers = substring.Or(
substring.Regexp(`(^|/)controls\.js$`), substring.Regexp(`(^|/)controls\.js$`),
substring.Regexp(`(^|/)dragdrop\.js$`), substring.Regexp(`(^|/)dragdrop\.js$`),
substring.Regexp(`(.*?)\.d\.ts$`), substring.Regexp(`(.*?)\.d\.ts$`),
substring.Regexp(`(^|/)mootools([^.]*)\d&#43;\.\d&#43;.\d&#43;([^.]*)\.js$`), substring.Regexp(`(^|/)mootools([^.]*)\d+\.\d+.\d+([^.]*)\.js$`),
substring.Regexp(`(^|/)dojo\.js$`), substring.Regexp(`(^|/)dojo\.js$`),
substring.Regexp(`(^|/)MochiKit\.js$`), substring.Regexp(`(^|/)MochiKit\.js$`),
substring.Regexp(`(^|/)yahoo-([^.]*)\.js$`), substring.Regexp(`(^|/)yahoo-([^.]*)\.js$`),
@ -80,16 +80,16 @@ var vendorMatchers = substring.Or(
substring.Regexp(`(^|/)fontello(.*?)\.css$`), substring.Regexp(`(^|/)fontello(.*?)\.css$`),
substring.Regexp(`(^|/)MathJax/`), substring.Regexp(`(^|/)MathJax/`),
substring.Regexp(`(^|/)Chart\.js$`), substring.Regexp(`(^|/)Chart\.js$`),
substring.Regexp(`(^|/)[Cc]ode[Mm]irror/(\d&#43;\.\d&#43;/)?(lib|mode|theme|addon|keymap|demo)`), substring.Regexp(`(^|/)[Cc]ode[Mm]irror/(\d+\.\d+/)?(lib|mode|theme|addon|keymap|demo)`),
substring.Regexp(`(^|/)shBrush([^.]*)\.js$`), substring.Regexp(`(^|/)shBrush([^.]*)\.js$`),
substring.Regexp(`(^|/)shCore\.js$`), substring.Regexp(`(^|/)shCore\.js$`),
substring.Regexp(`(^|/)shLegacy\.js$`), substring.Regexp(`(^|/)shLegacy\.js$`),
substring.Regexp(`(^|/)angular([^.]*)\.js$`), substring.Regexp(`(^|/)angular([^.]*)\.js$`),
substring.Regexp(`(^|\/)d3(\.v\d&#43;)?([^.]*)\.js$`), substring.Regexp(`(^|\/)d3(\.v\d+)?([^.]*)\.js$`),
substring.Regexp(`(^|/)react(-[^.]*)?\.js$`), substring.Regexp(`(^|/)react(-[^.]*)?\.js$`),
substring.Regexp(`(^|/)modernizr\-\d\.\d&#43;(\.\d&#43;)?\.js$`), substring.Regexp(`(^|/)modernizr\-\d\.\d+(\.\d+)?\.js$`),
substring.Regexp(`(^|/)modernizr\.custom\.\d&#43;\.js$`), substring.Regexp(`(^|/)modernizr\.custom\.\d+\.js$`),
substring.Regexp(`(^|/)knockout-(\d&#43;\.){3}(debug\.)?js$`), substring.Regexp(`(^|/)knockout-(\d+\.){3}(debug\.)?js$`),
substring.Regexp(`(^|/)docs?/_?(build|themes?|templates?|static)/`), substring.Regexp(`(^|/)docs?/_?(build|themes?|templates?|static)/`),
substring.Regexp(`(^|/)admin_media/`), substring.Regexp(`(^|/)admin_media/`),
substring.Regexp(`(^|/)env/`), substring.Regexp(`(^|/)env/`),
@ -117,7 +117,7 @@ var vendorMatchers = substring.Or(
substring.Regexp(`(^|/)jquery([^.]*)\.validate(\.unobtrusive)?\.js$`), substring.Regexp(`(^|/)jquery([^.]*)\.validate(\.unobtrusive)?\.js$`),
substring.Regexp(`(^|/)jquery([^.]*)\.unobtrusive\-ajax\.js$`), substring.Regexp(`(^|/)jquery([^.]*)\.unobtrusive\-ajax\.js$`),
substring.Regexp(`(^|/)[Mm]icrosoft([Mm]vc)?([Aa]jax|[Vv]alidation)(\.debug)?\.js$`), substring.Regexp(`(^|/)[Mm]icrosoft([Mm]vc)?([Aa]jax|[Vv]alidation)(\.debug)?\.js$`),
substring.Regexp(`^[Pp]ackages\/.&#43;\.\d&#43;\/`), substring.Regexp(`^[Pp]ackages\/.+\.\d+\/`),
substring.Regexp(`(^|/)extjs/.*?\.js$`), substring.Regexp(`(^|/)extjs/.*?\.js$`),
substring.Regexp(`(^|/)extjs/.*?\.xml$`), substring.Regexp(`(^|/)extjs/.*?\.xml$`),
substring.Regexp(`(^|/)extjs/.*?\.txt$`), substring.Regexp(`(^|/)extjs/.*?\.txt$`),