ralsina/tartrazine
1
0
Fork 0
mirror of https://github.com/ralsina/tartrazine.git synced 2024-12-05 00:00:34 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Squashed 'go-enry/' content from commit 7e3a9a7

Browse Source 
git-subtree-dir: go-enry
git-subtree-split: 7e3a9a7241
This commit is contained in:
Roberto Alsina 2024-09-04 16:33:41 -03:00
commit f955c625ad
192 changed files with 528500 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
.editorconfig Normal file
View File

@ -0,0 +1,22 @@
root = true
[*]
charset = utf-8
end_of_line = lf
indent_size = 2
indent_style = tab
insert_final_newline = true
trim_trailing_whitespace = true
[*.rb]
indent_style = space
[*.py]
indent_style = space
indent_size = 4
[_testdata/**]
charset = unset
indent_size = unset
indent_style = unset
trim_trailing_whitespace = unset

52
.github/workflows/goTest.yml vendored Normal file
View File

@ -0,0 +1,52 @@
on: [push, pull_request]
name: Go Tests
jobs:
test:
strategy:
matrix:
go-version: [1.18.x, 1.19.x]
platform: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.platform }}
steps:
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Set git on win to use LF
run: |
git config --global core.autocrlf false
git config --global core.eol lf
- name: Checkout code
uses: actions/checkout@v2
- name: Test
run: go test ./...
env:
ENRY_DEBUG: 1
test-oniguruma:
strategy:
matrix:
go-version: [1.18.x, 1.19.x]
runs-on: ubuntu-latest
env:
ONIGURUMA_VERSION: 6.9.4
steps:
- name: Install libonig5
run: |
wget "http://archive.ubuntu.com/ubuntu/pool/universe/libo/libonig/libonig5_${ONIGURUMA_VERSION}-1_amd64.deb"
sudo dpkg -i "libonig5_${ONIGURUMA_VERSION}-1_amd64.deb"
wget "http://archive.ubuntu.com/ubuntu/pool/universe/libo/libonig/libonig-dev_${ONIGURUMA_VERSION}-1_amd64.deb"
sudo dpkg -i "libonig-dev_${ONIGURUMA_VERSION}-1_amd64.deb"
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Checkout code
uses: actions/checkout@v2
- name: Test
run: go test -tags ${GO_TAGS} ./...
env:
GO_TAGS: oniguruma

32
.github/workflows/pyTest.yml vendored Normal file
View File

@ -0,0 +1,32 @@
on: [push, pull_request]
name: Python Tests
jobs:
test:
strategy:
matrix:
python-version: ['3.8', '3.9', '3.10']
platform: [ubuntu-latest, macos-latest]
runs-on: ${{ matrix.platform }}
steps:
- name: Install Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install Go
uses: actions/setup-go@v5
with:
go-version: 1.19.x
- name: Checkout code
uses: actions/checkout@v4
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r python/requirements.txt
if [ -f python/requirements.dev.txt ]; then pip install -r python/requirements.dev.txt; fi
- name: Build and install package
run: |
pip install setuptools wheel
pip -v install --no-use-pep517 -e python
- name: Test
run: |
pytest python/

130
.github/workflows/sync-linguist.yml vendored Normal file
View File

@ -0,0 +1,130 @@
name: Sync Linguist
on:
workflow_dispatch:
inputs:
linguist_tag:
description: 'Linguist tag override'
required: False
default: ''
schedule:
# Run once a day to check for new Linguist updates automatically
- cron: '0 20 * * *'
jobs:
sync-linguist:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v2
- uses: actions/checkout@v2
- name: Find previous Linguist commit
id: previous_linguist
run: |
set -euo pipefail
IFS=$'\n\t'
commit=$(sed --quiet --regexp-extended 's/[[:space:]]+commit[[:space:]]+=[[:space:]]"([a-f0-9]{40})"/\1/p' internal/code-generator/generator/generator_test.go)
echo "::set-output name=commit::$commit"
echo "::set-output name=short_commit::${commit::8}"
- uses: actions/checkout@v2
with:
repository: github/linguist
path: .linguist
fetch-depth: 0
- name: check out latest release
id: linguist-release
# the `grep -v "-"` is to exclude any pre-release versions.
# Linguist doesn't have any right now, but just in case.
run: |
set -euo pipefail
IFS=$'\n\t'
cd .linguist
if [[ -n "${{ github.event.inputs.linguist_tag }}" ]]; then
echo "Using tag override '${{ github.event.inputs.linguist_tag }}'"
latest="${{ github.event.inputs.linguist_tag }}"
else
latest=$(git tag --list | \
grep -v "-" | \
sort --version-sort --reverse | \
head -1)
fi
if [[ -z "$latest" ]]; then
echo "could not determine latest Linguist version"
exit 1
fi
echo "::set-output name=linguist_version::$latest"
git checkout $latest
commit=$(git rev-parse HEAD)
if [[ -z "$commit" ]]; then
echo "could not determine latest Linguist commit"
exit 1
fi
echo "::set-output name=commit::$commit"
echo "::set-output name=short_commit::${commit::8}"
cd ..
- name: Update Linguist information
run: |
set -euo pipefail
IFS=$'\n\t'
sed --in-place --regexp-extended 's/(commit[[:space:]]+=[[:space:]])("[a-f0-9]{40}")/\1"${{ steps.linguist-release.outputs.commit }}"/' internal/code-generator/generator/generator_test.go
sed --in-place --regexp-extended 's/version \*\*v.+\*\*\./version \*\*${{ steps.linguist-release.outputs.linguist_version }}\*\*\./' README.md
- name: Generate code
run: make code-generate
- name: Commit changes
id: commit
run: |
set -euo pipefail
IFS=$'\n\t'
echo "git current state:"
git status
branch_name="feature/sync-linguist-${{ steps.previous_linguist.outputs.short_commit }}"
if git rev-parse --quiet --verify $branch_name; then
echo "Linguist update branch $branch_name already exists"
echo "::set-output name=needs_pr::true"
exit 0
fi
if [[ -n "$(git status --porcelain)" ]]; then
echo "Creating branch $branch_name for PR"
git checkout -b $branch_name
echo "::set-output name=branch_name::$branch_name"
echo "Creating Linguist update commit"
git config user.name github-actions
git config user.email github-actions@github.com
git add .
git commit -m "Updated Linguist to ${{ steps.linguist-release.outputs.linguist_version }}"
git push --set-upstream origin $branch_name
echo "Changes committed. Will create PR."
echo "::set-output name=needs_pr::true"
exit 0
fi
echo "Linguist update unnecessary"
echo "::set-output name=needs_pr::false"
- name: Create Pull Request
id: open-pr
uses: repo-sync/pull-request@v2
if: ${{ steps.commit.outputs.needs_pr == 'true' }}
with:
source_branch: ${{ steps.commit.outputs.branch_name }}
pr_title: "Update Linguist to ${{ steps.linguist-release.outputs.linguist_version }}"
pr_body: |
Automated Linguist update :robot:
This PR updates Linguist from [${{ steps.previous_linguist.outputs.short_commit }}](https://github.com/github/linguist/commit/${{ steps.previous_linguist.outputs.commit }}) to [${{ steps.linguist-release.outputs.linguist_version }}](https://github.com/github/linguist/releases/tag/${{ steps.linguist-release.outputs.linguist_version }}) ([${{ steps.linguist-release.outputs.short_commit }}](https://github.com/github/linguist/commit/${{ steps.linguist-release.outputs.commit }}))
* [Linguist release notes](https://github.com/github/linguist/releases/tag/${{ steps.linguist-release.outputs.linguist_version }})
* [Compare Linguist code changes](https://github.com/github/linguist/compare/${{ steps.previous_linguist.outputs.commit }}...${{ steps.linguist-release.outputs.linguist_version }})
destination_branch: "master"
github_token: ${{ secrets.GITHUB_TOKEN }}
- name: output-url
if: ${{ steps.commit.outputs.needs_pr == 'true' }}
run: echo ${{ steps.open-pr.outputs.pr_url }}
- name: No PR Created
if: ${{ steps.commit.needs_pr != 'true' }}
run: echo "No changes for ${{ steps.linguist-release.outputs.linguist_version }}"

12
.gitignore vendored Normal file
View File

@ -0,0 +1,12 @@
.linguist*
benchmarks/output
.ci
Makefile.main
.shared
.idea
.docsrv-resources
build/
vendor/
java/lib/
.vscode/
.venv

202
LICENSE Normal file
View File

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

66
Makefile Normal file
View File

@ -0,0 +1,66 @@
LINGUIST_PATH = .linguist
# shared objects
RESOURCES_DIR=./.shared
LINUX_DIR=$(RESOURCES_DIR)/linux-x86-64
LINUX_SHARED_LIB=$(LINUX_DIR)/libenry.so
DARWIN_DIR=$(RESOURCES_DIR)/darwin
DARWIN_SHARED_LIB=$(DARWIN_DIR)/libenry.dylib
STATIC_LIB=$(RESOURCES_DIR)/libenry.a
HEADER_FILE=libenry.h
NATIVE_LIB=./shared/enry.go
$(LINGUIST_PATH):
git clone https://github.com/github/linguist.git $@
clean-linguist:
rm -rf $(LINGUIST_PATH)
clean-shared:
rm -rf $(RESOURCES_DIR)
clean: clean-linguist clean-shared
code-generate: $(LINGUIST_PATH)
mkdir -p data && \
go run internal/code-generator/main.go
ENRY_TEST_REPO="$${PWD}/.linguist" go test -v \
-run Test_GeneratorTestSuite \
./internal/code-generator/generator \
-testify.m TestUpdateGeneratorTestSuiteGold \
-update_gold
benchmarks: $(LINGUIST_PATH)
go test -run=NONE -bench=. && \
benchmarks/linguist-total.rb
benchmarks-samples: $(LINGUIST_PATH)
go test -run=NONE -bench=. -benchtime=5us && \
benchmarks/linguist-samples.rb
benchmarks-slow: $(LINGUIST_PATH)
mkdir -p benchmarks/output && \
go test -run=NONE -bench=. -slow -benchtime=100ms -timeout=100h > benchmarks/output/enry_samples.bench && \
benchmarks/linguist-samples.rb 5 > benchmarks/output/linguist_samples.bench
linux-shared: $(LINUX_SHARED_LIB)
darwin-shared: $(DARWIN_SHARED_LIB)
$(DARWIN_SHARED_LIB):
mkdir -p $(DARWIN_DIR) && \
CC="o64-clang" CXX="o64-clang++" CGO_ENABLED=1 GOOS=darwin go build -buildmode=c-shared -o $(DARWIN_SHARED_LIB) $(NATIVE_LIB) && \
mv $(DARWIN_DIR)/$(HEADER_FILE) $(RESOURCES_DIR)/$(HEADER_FILE)
$(LINUX_SHARED_LIB):
mkdir -p $(LINUX_DIR) && \
CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -buildmode=c-shared -o $(LINUX_SHARED_LIB) $(NATIVE_LIB) && \
mv $(LINUX_DIR)/$(HEADER_FILE) $(RESOURCES_DIR)/$(HEADER_FILE)
static: $(STATIC_LIB)
$(STATIC_LIB):
CGO_ENABLED=1 go build -buildmode=c-archive -o $(STATIC_LIB) $(NATIVE_LIB)
.PHONY: benchmarks benchmarks-samples benchmarks-slow

324
README.md Normal file
View File

@ -0,0 +1,324 @@
# go-enry [![GoDoc](https://godoc.org/github.com/go-enry/go-enry?status.svg)](https://pkg.go.dev/github.com/go-enry/go-enry/v2) [![Test](https://github.com/go-enry/go-enry/actions/workflows/goTest.yml/badge.svg)](https://github.com/go-enry/go-enry/actions/workflows/goTest.yml?query=branch%3Amaster)
Programming language detector and toolbox to ignore binary or vendored files. _enry_, started as a port to _Go_ of the original [Linguist](https://github.com/github/linguist) _Ruby_ library, that has an improved _2x performance_.
- [CLI](#cli)
- [Library](#library)
- [Use cases](#use-cases)
- [By filename](#by-filename)
- [By text](#by-text)
- [By file](#by-file)
- [Filtering](#filtering-vendoring-binaries-etc)
- [Coloring](#language-colors-and-groups)
- [Languages](#languages)
- [Go](#go)
- [Java bindings](#java-bindings)
- [Python bindings](#python-bindings)
- [Rust bindings](#rust-bindings)
- [Divergences from linguist](#divergences-from-linguist)
- [Benchmarks](#benchmarks)
- [Why Enry?](#why-enry)
- [Development](#development)
- [Sync with github/linguist upstream](#sync-with-githublinguist-upstream)
- [Misc](#misc)
- [License](#license)
# CLI
The CLI binary is hosted in a separate repository [go-enry/enry](https://github.com/go-enry/enry).
# Library
_enry_ is also a Go library for guessing a programming language that exposes API through FFI to multiple programming environments.
## Use cases
_enry_ guesses a programming language using a sequence of matching _strategies_ that are
applied progressively to narrow down the possible options. Each _strategy_ varies on the type
of input data that it needs to make a decision: file name, extension, the first line of the file, the full content of the file, etc.
Depending on available input data, enry API can be roughly divided into the next categories or use cases.
### By filename
Next functions require only a name of the file to make a guess:
- `GetLanguageByExtension` uses only file extension (wich may be ambiguous)
- `GetLanguageByFilename` useful for cases like `.gitignore`, `.bashrc`, etc
- all [filtering helpers](#filtering)
Please note that such guesses are expected not to be very accurate.
### By text
To make a guess only based on the content of the file or a text snippet, use
- `GetLanguageByShebang` reads only the first line of text to identify the [shebang](<https://en.wikipedia.org/wiki/Shebang_(Unix)>).
- `GetLanguageByModeline` for cases when Vim/Emacs modeline e.g. `/* vim: set ft=cpp: */` may be present at a head or a tail of the text.
- `GetLanguageByClassifier` uses a Bayesian classifier trained on all the `./samples/` from Linguist.
It usually is a last-resort strategy that is used to disambiguate the guess of the previous strategies, and thus it requires a list of "candidate" guesses. One can provide a list of all known languages - keys from the `data.LanguagesLogProbabilities` as possible candidates if more intelligent hypotheses are not available, at the price of possibly suboptimal accuracy.
### By file
The most accurate guess would be when both, a file name and it's content are available:
- `GetLanguagesByContent` only uses file extension and a set of regexp-based content heuristics.
- `GetLanguages` uses the full set of matching strategies and is expected to be most accurate.
### Filtering: vendoring, binaries, etc
_enry_ expose a set of file-level helpers `Is*` to simplify filtering out the files that are less interesting for the purpose of source code analysis:
- `IsBinary`
- `IsVendor`
- `IsConfiguration`
- `IsDocumentation`
- `IsDotFile`
- `IsImage`
- `IsTest`
- `IsGenerated`
### Language colors and groups
_enry_ exposes function to get language color to use for example in presenting statistics in graphs:
- `GetColor`
- `GetLanguageGroup` can be used to group similar languages together e.g. for `Less` this function will return `CSS`
## Languages
### Go
In a [Go module](https://github.com/golang/go/wiki/Modules),
import `enry` to the module by running:
```sh
go get github.com/go-enry/go-enry/v2
```
The rest of the examples will assume you have either done this or fetched the
library into your `GOPATH`.
```go
// The examples here and below assume you have imported the library.
import "github.com/go-enry/go-enry/v2"
lang, safe := enry.GetLanguageByExtension("foo.go")
fmt.Println(lang, safe)
// result: Go true
lang, safe := enry.GetLanguageByContent("foo.m", []byte("<matlab-code>"))
fmt.Println(lang, safe)
// result: Matlab true
lang, safe := enry.GetLanguageByContent("bar.m", []byte("<objective-c-code>"))
fmt.Println(lang, safe)
// result: Objective-C true
// all strategies together
lang := enry.GetLanguage("foo.cpp", []byte("<cpp-code>"))
// result: C++ true
```
Note that the returned boolean value `safe` is `true` if there is only one possible language detected.
A plural version of the same API allows getting a list of all possible languages for a given file.
```go
langs := enry.GetLanguages("foo.h", []byte("<cpp-code>"))
// result: []string{"C", "C++", "Objective-C}
langs := enry.GetLanguagesByExtension("foo.asc", []byte("<content>"), nil)
// result: []string{"AGS Script", "AsciiDoc", "Public Key"}
langs := enry.GetLanguagesByFilename("Gemfile", []byte("<content>"), []string{})
// result: []string{"Ruby"}
```
### Java bindings
Generated Java bindings using a C shared library and JNI are available under [`java`](https://github.com/go-enry/go-enry/blob/master/java).
A library is published on Maven as [tech.sourced:enry-java](https://mvnrepository.com/artifact/tech.sourced/enry-java) for macOS and linux platforms. Windows support is planned under [src-d/enry#150](https://github.com/src-d/enry/issues/150).
### Python bindings
Generated Python bindings using a C shared library and cffi are WIP under [src-d/enry#154](https://github.com/src-d/enry/issues/154).
A library is going to be published on pypi as [enry](https://pypi.org/project/enry/) for
macOS and linux platforms. Windows support is planned under [src-d/enry#150](https://github.com/src-d/enry/issues/150).
### Rust bindings
Generated Rust bindings using a C static library are available at https://github.com/go-enry/rs-enry.
## Divergences from Linguist
The `enry` library is based on the data from `github/linguist` version **v7.30.0**.
Parsing [linguist/samples](https://github.com/github/linguist/tree/master/samples) the following `enry` results are different from the Linguist:
- [Heuristic for ".plist" extension](https://github.com/github-linguist/linguist/blob/b5432ebc7e78f25415b98d48c2fbacddbf8df317/lib/linguist/heuristics.yml#L524) in 'XML Property List', due to unsupported backreference in RE2 regexp engine.
- [Heuristics for ".txt" extension](https://github.com/github/linguist/blob/8083cb5a89cee2d99f5a988f165994d0243f0d1e/lib/linguist/heuristics.yml#L521) in Vim Help File could not be parsed, due to unsupported negative lookahead in RE2 regexp engine.
- [Heuristics for ".sol" extension](https://github.com/github/linguist/blob/8083cb5a89cee2d99f5a988f165994d0243f0d1e/lib/linguist/heuristics.yml#L464) in Solidity could not be parsed, due to unsupported negative lookahead in RE2 regexp engine.
- [Heuristics for ".es" extension](https://github.com/github/linguist/blob/e761f9b013e5b61161481fcb898b59721ee40e3d/lib/linguist/heuristics.yml#L103) in JavaScript could not be parsed, due to unsupported backreference in RE2 regexp engine.
- [Heuristics for ".rno" extension](https://github.com/github/linguist/blob/3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d/lib/linguist/heuristics.yml#L365) in RUNOFF could not be parsed, due to unsupported lookahead in RE2 regexp engine.
- [Heuristics for ".inc" extension](https://github.com/github/linguist/blob/f0e2d0d7f1ce600b2a5acccaef6b149c87d8b99c/lib/linguist/heuristics.yml#L222) in NASL could not be parsed, due to unsupported possessive quantifier in RE2 regexp engine.
- [Heuristics for ".as" extension](https://github.com/github/linguist/blob/223c00bb80eff04788e29010f98c5778993d2b2a/lib/linguist/heuristics.yml#L67) in ActionScript could not be parsed, due to unsupported positive lookahead in RE2 regexp engine.
- [Heuristics for ".csc", ".gsc" and ".gsh" extension](https://github.com/github/linguist/blob/7469c7982d93f2ad922230d712f586a353dc1a42/lib/linguist/heuristics.yml#L650-L651) in GSC could not be parsed, due to unsupported non-backtracking subexpressions in RE2 regexp engine.
- [Heuristic for ".txt"](https://github.com/github/linguist/blob/bf853f1c663903e3ee35935189760191f1c45e1c/lib/linguist/heuristics.yml#L680-L702) detecting 'Adblock Filter List' regexp syntax not supported by RE2
- [IsVendor('bootstrap.css') == false](https://github.com/github/linguist/blob/v7.23.0/lib/linguist/vendor.yml#L77) v7.23 first unsupported RE syntax outside content heuristics
- As of [Linguist v5.3.2](https://github.com/github/linguist/releases/tag/v5.3.2) it is using [flex-based scanner in C for tokenization](https://github.com/github/linguist/pull/3846). Enry still uses [extract_token](https://github.com/github/linguist/pull/3846/files#diff-d5179df0b71620e3fac4535cd1368d15L60) regex-based algorithm. See [#193](https://github.com/src-d/enry/issues/193).
- Bayesian classifier can't distinguish "SQL" from "PLpgSQL. See [#194](https://github.com/src-d/enry/issues/194).
- Overriding languages and types though `.gitattributes` is not yet supported. See [#18](https://github.com/src-d/enry/issues/18).
- `enry` CLI output does NOT exclude `.gitignore`ed files and git submodules, as Linguist does
In all the cases above that have an issue number - we plan to update enry to match Linguist behavior.
> All the issues related to heuristics' regexp syntax incompatibilities with the RE2 engine can be avoided by using `oniguruma` instead (see [instuctions](#misc))
## Benchmarks
Enry's language detection has been compared with Linguist's on [_linguist/samples_](https://github.com/github/linguist/tree/master/samples).
We got these results:
![histogram](benchmarks/histogram/distribution.png)
The histogram shows the _number of files_ (y-axis) per _time interval bucket_ (x-axis).
Most of the files were detected faster by enry.
There are several cases where enry is slower than Linguist due to
Go regexp engine being slower than Ruby's on, wich is based on [oniguruma](https://github.com/kkos/oniguruma) library, written in C.
See [instructions](#misc) for running enry with oniguruma.
## Why Enry?
In the movie [My Fair Lady](https://en.wikipedia.org/wiki/My_Fair_Lady), [Professor Henry Higgins](http://www.imdb.com/character/ch0011719/) is a linguist who at the very beginning of the movie enjoys guessing the origin of people based on their accent.
"Enry Iggins" is how [Eliza Doolittle](http://www.imdb.com/character/ch0011720/), [pronounces](https://www.youtube.com/watch?v=pwNKyTktDIE) the name of the Professor.
## Development
To run the tests use:
go test ./...
Setting `ENRY_TEST_REPO` to a path to the existing checkout of the Linguist will avoid cloning it and speeds tests up.
Setting `ENRY_DEBUG=1` will provide insight into the Bayesian classifier built during `make code-generate`.
### Sync with github/linguist upstream
_enry_ re-uses parts of the original [github/linguist](https://github.com/github/linguist) to generate internal data structures.
In order to update to the latest release of linguist do:
```bash
$ git clone https://github.com/github/linguist.git .linguist
$ cd .linguist; git checkout <release-tag>; cd ..
# put the new release's commit sha in the generator_test.go (to re-generate .gold test fixtures)
# https://github.com/go-enry/go-enry/blob/13d3d66d37a87f23a013246a1b0678c9ee3d524b/internal/code-generator/generator/generator_test.go#L18
$ make code-generate
```
To stay in sync, enry needs to be updated when a new release of the linguist includes changes to any of the following files:
- [languages.yml](https://github.com/github/linguist/blob/master/lib/linguist/languages.yml)
- [heuristics.yml](https://github.com/github/linguist/blob/master/lib/linguist/heuristics.yml)
- [vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml)
- [documentation.yml](https://github.com/github/linguist/blob/master/lib/linguist/documentation.yml)
There now is automation for detecting the changes in the upstream Linguist project: every day Github CI runs [a job](.github/workflows/sync-linguist.yml) that will create a PR to this repo for each new Linguist release. It will include all the steps from the above.
When submitting a pull request syncing up to a new release manually, please make sure it only contains the changes in
the generated files (in [data](https://github.com/go-enry/go-enry/blob/master/data) subdirectory).
Separating all the necessary "manual" code changes to a different PR that includes some background description and an update to the documentation on ["divergences from linguist"](#divergences-from-linguist) is encouraged and very much appreciated, as it simplifies the maintenance (review/release notes/etc).
## Misc
<details>
<summary>Running a benchmark & faster regexp engine</summary>
### Benchmark
All benchmark scripts are in [_benchmarks_](https://github.com/go-enry/go-enry/blob/master/benchmarks) directory.
#### Dependencies
As benchmarks depend on Ruby and GitHub-Linguist gem make sure you have:
- Ruby (e.g using [`rbenv`](https://github.com/rbenv/rbenv)), [`bundler`](https://bundler.io/) installed
- Docker
- [native dependencies](https://github.com/github/linguist/#dependencies) installed
- Build the gem `cd .linguist && bundle install && rake build_gem && cd -`
- Install it `gem install --no-rdoc --no-ri --local .linguist/github-linguist-*.gem`
#### Quick benchmark
To run quicker benchmarks
make benchmarks
to get average times for the primary detection function and strategies for the whole samples set. If you want to see measures per sample file use:
make benchmarks-samples
#### Full benchmark
If you want to reproduce the same benchmarks as reported above:
- Make sure all [dependencies](#benchmark-dependencies) are installed
- Install [gnuplot](http://gnuplot.info) (in order to plot the histogram)
- Run `ENRY_TEST_REPO="$PWD/.linguist" benchmarks/run.sh` (takes ~15h)
It will run the benchmarks for enry and Linguist, parse the output, create csv files and plot the histogram.
### Faster regexp engine (optional)
[Oniguruma](https://github.com/kkos/oniguruma) is CRuby's regular expression engine.
It is very fast and performs better than the one built into Go runtime. _enry_ supports swapping
between those two engines thanks to [rubex](https://github.com/moovweb/rubex) project.
The typical overall speedup from using Oniguruma is 1.5-2x. However, it requires CGo and the external shared library.
On macOS with [Homebrew](https://brew.sh/), it is:
```
brew install oniguruma
```
On Ubuntu, it is
```
sudo apt install libonig-dev
```
To build enry with Oniguruma regexps use the `oniguruma` build tag
```
go get -v -t --tags oniguruma ./...
```
and then rebuild the project.
</details>
## License
Apache License, Version 2.0. See [LICENSE](LICENSE)

42
_testdata/C/image.c Normal file
View File

@ -0,0 +1,42 @@
/* GIMP RGB C-Source image dump (image.c) */
static const struct {
guint width;
guint height;
guint bytes_per_pixel; /* 2:RGB16, 3:RGB, 4:RGBA */
guint8 pixel_data[16 * 16 * 3 + 1];
} gimp_image = {
16, 16, 3,
"\377\063\000\377T\001\377\207\001\377\272\000\377\354\000\341\377\001\256\377\000{\377\001"
"Z\377\000(\377\000\000\377\014\001\377>\000\377q\001\377\243\000\377\325\001\377\367\377T"
"\001\377\207\001\377\271\000\377\354\000\341\377\001\256\377\001{\377\001Z\377\001(\377\001"
"\000\377\014\001\377>\001\377q\001\377\243\000\377\325\001\377\367\001\325\377\377\207\000"
"\377\272\001\377\354\000\341\377\001\257\377\001|\377\000[\377\001(\377\000\001\377\013\001"
"\377>\000\377q\000\377\244\000\377\325\000\377\367\000\325\377\001\243\377\377\272\001"
"\377\354\001\340\377\001\256\377\000|\377\001Z\377\000(\377\000\001\377\014\001\377>\000\377"
"q\000\377\243\001\377\326\000\377\370\001\325\377\001\243\377\000q\377\377\354\000\341"
"\377\001\257\377\000|\377\000Z\377\001(\377\000\000\377\013\000\377?\001\377q\001\377\243\000"
"\377\325\001\377\370\000\326\377\000\243\377\001q\377\000>\377\340\377\001\257\377\001"
"{\377\001Z\377\000(\377\001\000\377\014\001\377>\001\377q\000\377\243\000\377\325\000\377\367"
"\001\325\377\000\243\377\000p\377\001?\377\000\014\377\256\377\000|\377\000Z\377\001'\377"
"\001\001\377\014\001\377>\000\377p\001\377\243\001\377\326\001\377\367\000\326\377\000\243\377"
"\000q\377\000>\377\000\014\377(\000\377|\377\001J\377\001(\377\001\001\377\013\000\377>\001\377"
"q\000\377\243\000\377\325\000\377\367\001\326\377\000\243\377\000p\377\000>\377\001\014\377"
"'\000\377J\001\377I\377\001'\377\000\001\377\013\001\377>\001\377p\000\377\243\000\377\326\001"
"\367\377\000\326\377\000\243\377\000q\377\001>\377\001\014\377(\001\377I\001\377|\001\377"
"'\377\001\000\377\014\000\377>\001\377p\000\377\243\000\377\326\000\367\377\000\325\377\000"
"\243\377\001p\377\001?\377\001\014\377(\000\377Z\000\377|\000\377\256\000\377\001\377\013\000"
"\377>\001\377p\000\377\243\001\377\325\001\367\377\001\326\377\000\243\377\000p\377\001?"
"\377\001\013\377(\000\377Z\000\377|\000\377\256\000\377\341\001\377\000\377?\001\377p\001\377"
"\243\001\377\326\000\367\377\001\326\377\001\243\377\001q\377\001>\377\000\014\377(\000\377"
"Z\001\377|\000\377\256\001\377\341\000\377\377\000\354\000\377p\000\377\243\001\377\326\001"
"\367\377\001\326\377\000\243\377\001q\377\000>\377\001\013\377(\001\377Z\000\377|\001\377"
"\257\000\377\341\000\377\377\000\354\377\001\271\000\377\243\000\377\326\001\367\377\001"
"\326\377\001\243\377\000q\377\000>\377\000\014\377'\001\377Z\000\377{\001\377\256\000\377"
"\341\001\377\377\001\354\377\001\271\377\000\207\000\377\325\001\367\377\000\326\377\000"
"\243\377\001q\377\001>\377\001\014\377(\000\377Z\001\377|\000\377\256\000\377\340\000\377"
"\377\000\354\377\001\271\377\000\207\377\000T\000\367\377\001\325\377\000\243\377\000p\377"
"\001>\377\001\014\377(\001\377Z\000\377|\001\377\256\000\377\341\001\377\377\000\354\377\000"
"\271\377\000\207\377\000T\377\001\063",
};

31
_testdata/C/image.h Normal file
View File

@ -0,0 +1,31 @@
/* GIMP header image file format (RGB): image.h */
static unsigned int width = 16;
static unsigned int height = 16;
/* Call this macro repeatedly. After each use, the pixel data can be extracted */
#define HEADER_PIXEL(data,pixel) {\
pixel[0] = (((data[0] - 33) << 2) | ((data[1] - 33) >> 4)); \
pixel[1] = ((((data[1] - 33) & 0xF) << 4) | ((data[2] - 33) >> 2)); \
pixel[2] = ((((data[2] - 33) & 0x3) << 6) | ((data[3] - 33))); \
data += 4; \
}
static char *header_data =
"`T-!`V1\"`Y=\"`\\I!`_Q!Y@]\"LP]!?`]\"7P]!+0]!!0]-!@]_!0^R!@_D!0`6!@`X"
"`V1\"`Y=\"`\\E!`_Q!Y@]\"LP]\"?`]\"7P]\"+0]\"!0]-!@]_!@^R!@_D!0`6!@`X!>8`"
"`Y=!`\\I\"`_Q!Y@]\"L`]\"@0]!7`]\"+0]!!@],!@]_!0^R!0_E!0`6!0`X!.8`!;0`"
"`\\I\"`_Q\"Y0]\"LP]!@0]\"7P]!+0]!!@]-!@]_!0^R!0_D!@`7!0`Y!>8`!;0`!((`"
"`_Q!Y@]\"L`]!@0]!7P]\"+0]!!0],!0]`!@^R!@_D!0`6!@`Y!.<`!+0`!8(`!$\\`"
"Y0]\"L`]\"?`]\"7P]!+0]\"!0]-!@]_!@^R!0_D!0`6!0`X!>8`!+0`!($`!4``!!T`"
"LP]!@0]!7P]\"*`]\"!@]-!@]_!0^Q!@_D!@`7!@`X!.<`!+0`!((`!$\\`!!T`+!$`"
"@0]\"3P]\"+0]\"!@],!0]_!@^R!0_D!0`6!0`X!><`!+0`!($`!$\\`!1T`*Q$`3A(`"
"3@]\"*`]!!@],!@]_!@^Q!0_D!0`7!@@`!.<`!+0`!((`!4\\`!1T`+!(`31(`@!(`"
"*`]\"!0]-!0]_!@^Q!0_D!0`7!0@`!.8`!+0`!8$`!4``!1T`+!$`7A$`@!$`LA$`"
"!@],!0]_!@^Q!0_D!@`6!@@`!><`!+0`!($`!4``!1P`+!$`7A$`@!$`LA$`Y1(`"
"!0]`!@^Q!@_D!@`7!0@`!><`!;0`!8(`!4\\`!!T`+!$`7A(`@!$`LA(`Y1$``Q$M"
"!0^Q!0_D!@`7!@@`!><`!+0`!8(`!$\\`!1P`+!(`7A$`@!(`LQ$`Y1$``Q$M`Q'Z"
"!0_D!0`7!@@`!><`!;0`!((`!$\\`!!T`*Q(`7A$`?Q(`LA$`Y1(``Q(M`Q'Z`Q#("
"!0`6!@@`!.<`!+0`!8(`!4\\`!1T`+!$`7A(`@!$`LA$`Y!$``Q$M`Q'Z`Q#(`Q\"5"
"!0@`!>8`!+0`!($`!4\\`!1T`+!(`7A$`@!(`LA$`Y1(``Q$M`Q#Z`Q#(`Q\"5`Q%T"
"";

1
_testdata/Data/bootstrap.css.map Normal file
View File

File diff suppressed because one or more lines are too long

12
_testdata/Data/sourcemap.v1.map Normal file
View File

@ -0,0 +1,12 @@
/** Begin line maps. **/{ “file”:”out.js”, "count": 2 }
[0,0,0,0,0,0,1,1,1,1,2]
[2,2,2,2,2,2,3,4,4,4,4,4]
/** Begin file information. **/
[“a.js”, “b.js”]
[“b.js”, “c.js”, “d.js”]
/** Begin mapping definitions. **/
["a.js", 1, 34]
["a.js", 5, 2]
["b.js", 1, 3, "event"]
["c.js", 1, 4]
["d.js", 3, 78, "foo"]

1
_testdata/Data/sourcemap.v3.map Normal file
View File

@ -0,0 +1 @@
{"version":3,"file":"out.js","sourceRoot":"","sources":["foo.js","bar.js"],"sourcesContent":[null,null],"names":["src","maps","are","fun"],"mappings":"A,AAAB;;ABCDE;"}

86
_testdata/Generated/Haxe/Main.cpp Normal file
View File

@ -0,0 +1,86 @@
// Generated by Haxe 4.0.5
#include <hxcpp.h>
#ifndef INCLUDED_Main
#include <Main.h>
#endif
#ifndef INCLUDED_haxe_Log
#include <haxe/Log.h>
#endif
HX_LOCAL_STACK_FRAME(_hx_pos_e47a9afac0942eb9_3_main,"Main","main",0xed0e206e,"Main.main","Main.hx",3,0x087e5c05)
void Main_obj::__construct() { }
Dynamic Main_obj::__CreateEmpty() { return new Main_obj; }
void *Main_obj::_hx_vtable = 0;
Dynamic Main_obj::__Create(hx::DynamicArray inArgs)
{
hx::ObjectPtr< Main_obj > _hx_result = new Main_obj();
_hx_result->__construct();
return _hx_result;
}
bool Main_obj::_hx_isInstanceOf(int inClassId) {
return inClassId==(int)0x00000001 || inClassId==(int)0x332f6459;
}
void Main_obj::main(){
HX_STACKFRAME(&_hx_pos_e47a9afac0942eb9_3_main)
HXDLIN( 3) ::haxe::Log_obj::trace(HX_("Hello World",84,f6,db,6f),hx::SourceInfo(HX_("source/Main.hx",91,d3,a7,40),3,HX_("Main",59,64,2f,33),HX_("main",39,38,56,48)));
}
STATIC_HX_DEFINE_DYNAMIC_FUNC0(Main_obj,main,(void))
Main_obj::Main_obj()
{
}
bool Main_obj::__GetStatic(const ::String &inName, Dynamic &outValue, hx::PropertyAccess inCallProp)
{
switch(inName.length) {
case 4:
if (HX_FIELD_EQ(inName,"main") ) { outValue = main_dyn(); return true; }
}
return false;
}
#ifdef HXCPP_SCRIPTABLE
static hx::StorageInfo *Main_obj_sMemberStorageInfo = 0;
static hx::StaticInfo *Main_obj_sStaticStorageInfo = 0;
#endif
hx::Class Main_obj::__mClass;
static ::String Main_obj_sStaticFields[] = {
HX_("main",39,38,56,48),
::String(null())
};
void Main_obj::__register()
{
Main_obj _hx_dummy;
Main_obj::_hx_vtable = *(void **)&_hx_dummy;
hx::Static(__mClass) = new hx::Class_obj();
__mClass->mName = HX_("Main",59,64,2f,33);
__mClass->mSuper = &super::__SGetClass();
__mClass->mConstructEmpty = &__CreateEmpty;
__mClass->mConstructArgs = &__Create;
__mClass->mGetStaticField = &Main_obj::__GetStatic;
__mClass->mSetStaticField = &hx::Class_obj::SetNoStaticField;
__mClass->mStatics = hx::Class_obj::dupFunctions(Main_obj_sStaticFields);
__mClass->mMembers = hx::Class_obj::dupFunctions(0 /* sMemberFields */);
__mClass->mCanCast = hx::TCanCast< Main_obj >;
#ifdef HXCPP_SCRIPTABLE
__mClass->mMemberStorageInfo = Main_obj_sMemberStorageInfo;
#endif
#ifdef HXCPP_SCRIPTABLE
__mClass->mStaticStorageInfo = Main_obj_sStaticStorageInfo;
#endif
hx::_hx_RegisterClass(__mClass->mName, __mClass);
}

37
_testdata/Generated/Haxe/Main.cs Normal file
View File

@ -0,0 +1,37 @@
// Generated by Haxe 4.0.5
#pragma warning disable 109, 114, 219, 429, 168, 162
public class EntryPoint__Main {
public static void Main() {
global::cs.Boot.init();
{
global::Main.main();
}
}
}
public class Main : global::haxe.lang.HxObject {
public Main(global::haxe.lang.EmptyObject empty) {
}
public Main() {
global::Main.__hx_ctor__Main(this);
}
protected static void __hx_ctor__Main(global::Main __hx_this) {
}
public static void main() {
unchecked {
global::haxe.Log.trace.__hx_invoke2_o(default(double), "Hello World", default(double), new global::haxe.lang.DynamicObject(new int[]{302979532, 1547539107, 1648581351}, new object[]{"main", "Main", "source/Main.hx"}, new int[]{1981972957}, new double[]{((double) (3) )}));
}
}
}

58
_testdata/Generated/Haxe/Main.h Normal file
View File

@ -0,0 +1,58 @@
// Generated by Haxe 4.0.5
#ifndef INCLUDED_Main
#define INCLUDED_Main
#ifndef HXCPP_H
#include <hxcpp.h>
#endif
HX_DECLARE_CLASS0(Main)
class HXCPP_CLASS_ATTRIBUTES Main_obj : public hx::Object
{
public:
typedef hx::Object super;
typedef Main_obj OBJ_;
Main_obj();
public:
enum { _hx_ClassId = 0x332f6459 };
void __construct();
inline void *operator new(size_t inSize, bool inContainer=false,const char *inName="Main")
{ return hx::Object::operator new(inSize,inContainer,inName); }
inline void *operator new(size_t inSize, int extra)
{ return hx::Object::operator new(inSize+extra,false,"Main"); }
inline static hx::ObjectPtr< Main_obj > __new() {
hx::ObjectPtr< Main_obj > __this = new Main_obj();
__this->__construct();
return __this;
}
inline static hx::ObjectPtr< Main_obj > __alloc(hx::Ctx *_hx_ctx) {
Main_obj *__this = (Main_obj*)(hx::Ctx::alloc(_hx_ctx, sizeof(Main_obj), false, "Main"));
*(void **)__this = Main_obj::_hx_vtable;
return __this;
}
static void * _hx_vtable;
static Dynamic __CreateEmpty();
static Dynamic __Create(hx::DynamicArray inArgs);
//~Main_obj();
HX_DO_RTTI_ALL;
static bool __GetStatic(const ::String &inString, Dynamic &outValue, hx::PropertyAccess inCallProp);
static void __register();
bool _hx_isInstanceOf(int inClassId);
::String __ToString() const { return HX_("Main",59,64,2f,33); }
static void main();
static ::Dynamic main_dyn();
};
#endif /* INCLUDED_Main */

41
_testdata/Generated/Haxe/Main.java Normal file
View File

@ -0,0 +1,41 @@
// Generated by Haxe 4.0.5
package haxe.root;
import haxe.root.*;
@SuppressWarnings(value={"rawtypes", "unchecked"})
public class Main extends haxe.lang.HxObject
{
public static void main(String[] args)
{
haxe.java.Init.init();
{
haxe.root.Main.main();
}
}
public Main(haxe.lang.EmptyObject empty)
{
}
public Main()
{
haxe.root.Main.__hx_ctor__Main(this);
}
protected static void __hx_ctor__Main(haxe.root.Main __hx_this)
{
}
public static void main()
{
haxe.Log.trace.__hx_invoke2_o(0.0, "Hello World", 0.0, new haxe.lang.DynamicObject(new java.lang.String[]{"className", "fileName", "methodName"}, new java.lang.Object[]{"Main", "source/Main.hx", "main"}, new java.lang.String[]{"lineNumber"}, new double[]{((double) (((double) (3) )) )}));
}
}

31
_testdata/Generated/Haxe/Main.php Normal file
View File

@ -0,0 +1,31 @@
<?php
/**
* Generated by Haxe 4.0.5
*/
use \php\_Boot\HxAnon;
use \php\Boot;
use \haxe\Log;
class Main {
/**
* @return void
*/
public static function main () {
#source/Main.hx:3: characters 3-8
(Log::$trace)("Hello World", new HxAnon([
"fileName" => "source/Main.hx",
"lineNumber" => 3,
"className" => "Main",
"methodName" => "main",
]));
}
/**
* @return void
*/
public function __construct () {
}
}
Boot::registerClass(Main::class, 'Main');

8
_testdata/Generated/Haxe/main.js Normal file
View File

@ -0,0 +1,8 @@
// Generated by Haxe 4.0.5
(function ($global) { "use strict";
var Main = function() { };
Main.main = function() {
console.log("source/Main.hx:3:","Hello World");
};
Main.main();
})({});

816
_testdata/Generated/Haxe/main.lua Normal file
View File

@ -0,0 +1,816 @@
-- Generated by Haxe 4.0.5
local _hx_array_mt = {
__newindex = function(t,k,v)
local len = t.length
t.length = k >= len and (k + 1) or len
rawset(t,k,v)
end
}
local function _hx_tab_array(tab,length)
tab.length = length
return setmetatable(tab, _hx_array_mt)
end
local function _hx_anon_newindex(t,k,v) t.__fields__[k] = true; rawset(t,k,v); end
local _hx_anon_mt = {__newindex=_hx_anon_newindex}
local function _hx_a(...)
local __fields__ = {};
local ret = {__fields__ = __fields__};
local max = select('#',...);
local tab = {...};
local cur = 1;
while cur < max do
local v = tab[cur];
__fields__[v] = true;
ret[v] = tab[cur+1];
cur = cur + 2
end
return setmetatable(ret, _hx_anon_mt)
end
local function _hx_e()
return setmetatable({__fields__ = {}}, _hx_anon_mt)
end
local function _hx_o(obj)
return setmetatable(obj, _hx_anon_mt)
end
local function _hx_new(prototype)
return setmetatable({__fields__ = {}}, {__newindex=_hx_anon_newindex, __index=prototype})
end
local _hxClasses = {}
local Int = _hx_e();
local Dynamic = _hx_e();
local Float = _hx_e();
local Bool = _hx_e();
local Class = _hx_e();
local Enum = _hx_e();
local Array = _hx_e()
__lua_lib_luautf8_Utf8 = _G.require("lua-utf8")
local Main = _hx_e()
local Math = _hx_e()
local String = _hx_e()
local Std = _hx_e()
__haxe_Log = _hx_e()
__lua_Boot = _hx_e()
local _hx_bind, _hx_bit, _hx_staticToInstance, _hx_funcToField, _hx_maxn, _hx_print, _hx_apply_self, _hx_box_mr, _hx_bit_clamp, _hx_table, _hx_bit_raw
local _hx_pcall_default = {};
local _hx_pcall_break = {};
Array.new = function()
local self = _hx_new(Array.prototype)
Array.super(self)
return self
end
Array.super = function(self)
_hx_tab_array(self, 0);
end
Array.prototype = _hx_a();
Array.prototype.concat = function(self,a)
local _g = _hx_tab_array({}, 0);
local _g1 = 0;
local _g2 = self;
while (_g1 < _g2.length) do
local i = _g2[_g1];
_g1 = _g1 + 1;
_g:push(i);
end;
local ret = _g;
local _g3 = 0;
while (_g3 < a.length) do
local i1 = a[_g3];
_g3 = _g3 + 1;
ret:push(i1);
end;
do return ret end
end
Array.prototype.join = function(self,sep)
local tbl = ({});
local _gthis = self;
local cur_length = 0;
local i = _hx_o({__fields__={hasNext=true,next=true},hasNext=function(self)
do return cur_length < _gthis.length end;
end,next=function(self)
cur_length = cur_length + 1;
do return _gthis[cur_length - 1] end;
end});
while (i:hasNext()) do
local i1 = i:next();
_G.table.insert(tbl, Std.string(i1));
end;
do return _G.table.concat(tbl, sep) end
end
Array.prototype.pop = function(self)
if (self.length == 0) then
do return nil end;
end;
local ret = self[self.length - 1];
self[self.length - 1] = nil;
self.length = self.length - 1;
do return ret end
end
Array.prototype.push = function(self,x)
self[self.length] = x;
do return self.length end
end
Array.prototype.reverse = function(self)
local tmp;
local i = 0;
while (i < Std.int(self.length / 2)) do
tmp = self[i];
self[i] = self[(self.length - i) - 1];
self[(self.length - i) - 1] = tmp;
i = i + 1;
end;
end
Array.prototype.shift = function(self)
if (self.length == 0) then
do return nil end;
end;
local ret = self[0];
if (self.length == 1) then
self[0] = nil;
else
if (self.length > 1) then
self[0] = self[1];
_G.table.remove(self, 1);
end;
end;
local tmp = self;
tmp.length = tmp.length - 1;
do return ret end
end
Array.prototype.slice = function(self,pos,_end)
if ((_end == nil) or (_end > self.length)) then
_end = self.length;
else
if (_end < 0) then
_end = _G.math.fmod((self.length - (_G.math.fmod(-_end, self.length))), self.length);
end;
end;
if (pos < 0) then
pos = _G.math.fmod((self.length - (_G.math.fmod(-pos, self.length))), self.length);
end;
if ((pos > _end) or (pos > self.length)) then
do return _hx_tab_array({}, 0) end;
end;
local ret = _hx_tab_array({}, 0);
local _g = pos;
local _g1 = _end;
while (_g < _g1) do
_g = _g + 1;
local i = _g - 1;
ret:push(self[i]);
end;
do return ret end
end
Array.prototype.sort = function(self,f)
local i = 0;
local l = self.length;
while (i < l) do
local swap = false;
local j = 0;
local max = (l - i) - 1;
while (j < max) do
if (f(self[j], self[j + 1]) > 0) then
local tmp = self[j + 1];
self[j + 1] = self[j];
self[j] = tmp;
swap = true;
end;
j = j + 1;
end;
if (not swap) then
break;
end;
i = i + 1;
end;
end
Array.prototype.splice = function(self,pos,len)
if ((len < 0) or (pos > self.length)) then
do return _hx_tab_array({}, 0) end;
else
if (pos < 0) then
pos = self.length - (_G.math.fmod(-pos, self.length));
end;
end;
len = Math.min(len, self.length - pos);
local ret = _hx_tab_array({}, 0);
local _g = pos;
local _g1 = pos + len;
while (_g < _g1) do
_g = _g + 1;
local i = _g - 1;
ret:push(self[i]);
self[i] = self[i + len];
end;
local _g2 = pos + len;
local _g3 = self.length;
while (_g2 < _g3) do
_g2 = _g2 + 1;
local i1 = _g2 - 1;
self[i1] = self[i1 + len];
end;
local tmp = self;
tmp.length = tmp.length - len;
do return ret end
end
Array.prototype.toString = function(self)
local tbl = ({});
_G.table.insert(tbl, "[");
_G.table.insert(tbl, self:join(","));
_G.table.insert(tbl, "]");
do return _G.table.concat(tbl, "") end
end
Array.prototype.unshift = function(self,x)
local len = self.length;
local _g = 0;
local _g1 = len;
while (_g < _g1) do
_g = _g + 1;
local i = _g - 1;
self[len - i] = self[(len - i) - 1];
end;
self[0] = x;
end
Array.prototype.insert = function(self,pos,x)
if (pos > self.length) then
pos = self.length;
end;
if (pos < 0) then
pos = self.length + pos;
if (pos < 0) then
pos = 0;
end;
end;
local cur_len = self.length;
while (cur_len > pos) do
self[cur_len] = self[cur_len - 1];
cur_len = cur_len - 1;
end;
self[pos] = x;
end
Array.prototype.remove = function(self,x)
local _g = 0;
local _g1 = self.length;
while (_g < _g1) do
_g = _g + 1;
local i = _g - 1;
if (self[i] == x) then
local _g2 = i;
local _g11 = self.length - 1;
while (_g2 < _g11) do
_g2 = _g2 + 1;
local j = _g2 - 1;
self[j] = self[j + 1];
end;
self[self.length - 1] = nil;
self.length = self.length - 1;
do return true end;
end;
end;
do return false end
end
Array.prototype.indexOf = function(self,x,fromIndex)
local _end = self.length;
if (fromIndex == nil) then
fromIndex = 0;
else
if (fromIndex < 0) then
fromIndex = self.length + fromIndex;
if (fromIndex < 0) then
fromIndex = 0;
end;
end;
end;
local _g = fromIndex;
local _g1 = _end;
while (_g < _g1) do
_g = _g + 1;
local i = _g - 1;
if (x == self[i]) then
do return i end;
end;
end;
do return -1 end
end
Array.prototype.lastIndexOf = function(self,x,fromIndex)
if ((fromIndex == nil) or (fromIndex >= self.length)) then
fromIndex = self.length - 1;
else
if (fromIndex < 0) then
fromIndex = self.length + fromIndex;
if (fromIndex < 0) then
do return -1 end;
end;
end;
end;
local i = fromIndex;
while (i >= 0) do
if (self[i] == x) then
do return i end;
else
i = i - 1;
end;
end;
do return -1 end
end
Array.prototype.copy = function(self)
local _g = _hx_tab_array({}, 0);
local _g1 = 0;
local _g2 = self;
while (_g1 < _g2.length) do
local i = _g2[_g1];
_g1 = _g1 + 1;
_g:push(i);
end;
do return _g end
end
Array.prototype.map = function(self,f)
local _g = _hx_tab_array({}, 0);
local _g1 = 0;
local _g2 = self;
while (_g1 < _g2.length) do
local i = _g2[_g1];
_g1 = _g1 + 1;
_g:push(f(i));
end;
do return _g end
end
Array.prototype.filter = function(self,f)
local _g = _hx_tab_array({}, 0);
local _g1 = 0;
local _g2 = self;
while (_g1 < _g2.length) do
local i = _g2[_g1];
_g1 = _g1 + 1;
if (f(i)) then
_g:push(i);
end;
end;
do return _g end
end
Array.prototype.iterator = function(self)
local _gthis = self;
local cur_length = 0;
do return _hx_o({__fields__={hasNext=true,next=true},hasNext=function(self)
do return cur_length < _gthis.length end;
end,next=function(self)
cur_length = cur_length + 1;
do return _gthis[cur_length - 1] end;
end}) end
end
Array.prototype.resize = function(self,len)
if (self.length < len) then
self.length = len;
else
if (self.length > len) then
local _g = len;
local _g1 = self.length;
while (_g < _g1) do
_g = _g + 1;
local i = _g - 1;
self[i] = nil;
end;
self.length = len;
end;
end;
end
Main.new = {}
Main.main = function()
__haxe_Log.trace("Hello World", _hx_o({__fields__={fileName=true,lineNumber=true,className=true,methodName=true},fileName="source/Main.hx",lineNumber=3,className="Main",methodName="main"}));
end
Math.new = {}
Math.isNaN = function(f)
do return f ~= f end;
end
Math.isFinite = function(f)
if (f > -_G.math.huge) then
do return f < _G.math.huge end;
else
do return false end;
end;
end
Math.min = function(a,b)
if (Math.isNaN(a) or Math.isNaN(b)) then
do return (0/0) end;
else
do return _G.math.min(a, b) end;
end;
end
String.new = function(string)
local self = _hx_new(String.prototype)
String.super(self,string)
self = string
return self
end
String.super = function(self,string)
end
String.__index = function(s,k)
if (k == "length") then
do return __lua_lib_luautf8_Utf8.len(s) end;
else
local o = String.prototype;
local field = k;
if ((function()
local _hx_1
if ((_G.type(o) == "string") and ((String.prototype[field] ~= nil) or (field == "length"))) then
_hx_1 = true; elseif (o.__fields__ ~= nil) then
_hx_1 = o.__fields__[field] ~= nil; else
_hx_1 = o[field] ~= nil; end
return _hx_1
end )()) then
do return String.prototype[k] end;
else
if (String.__oldindex ~= nil) then
if (_G.type(String.__oldindex) == "function") then
do return String.__oldindex(s, k) end;
else
if (_G.type(String.__oldindex) == "table") then
do return String.__oldindex[k] end;
end;
end;
do return nil end;
else
do return nil end;
end;
end;
end;
end
String.fromCharCode = function(code)
do return __lua_lib_luautf8_Utf8.char(code) end;
end
String.prototype = _hx_a();
String.prototype.toUpperCase = function(self)
do return __lua_lib_luautf8_Utf8.upper(self) end
end
String.prototype.toLowerCase = function(self)
do return __lua_lib_luautf8_Utf8.lower(self) end
end
String.prototype.indexOf = function(self,str,startIndex)
if (startIndex == nil) then
startIndex = 1;
else
startIndex = startIndex + 1;
end;
local r = __lua_lib_luautf8_Utf8.find(self, str, startIndex, true);
if ((r ~= nil) and (r > 0)) then
do return r - 1 end;
else
do return -1 end;
end;
end
String.prototype.lastIndexOf = function(self,str,startIndex)
local i = 0;
local ret = -1;
if (startIndex == nil) then
startIndex = __lua_lib_luautf8_Utf8.len(self);
end;
while (true) do
local startIndex1 = ret + 1;
if (startIndex1 == nil) then
startIndex1 = 1;
else
startIndex1 = startIndex1 + 1;
end;
local r = __lua_lib_luautf8_Utf8.find(self, str, startIndex1, true);
local p = (function()
local _hx_1
if ((r ~= nil) and (r > 0)) then
_hx_1 = r - 1; else
_hx_1 = -1; end
return _hx_1
end )();
if ((p == -1) or (p > startIndex)) then
break;
end;
ret = p;
end;
do return ret end
end
String.prototype.split = function(self,delimiter)
local idx = 1;
local ret = _hx_tab_array({}, 0);
local delim_offset = (function()
local _hx_1
if (__lua_lib_luautf8_Utf8.len(delimiter) > 0) then
_hx_1 = __lua_lib_luautf8_Utf8.len(delimiter); else
_hx_1 = 1; end
return _hx_1
end )();
while (idx ~= nil) do
local newidx = 0;
if (__lua_lib_luautf8_Utf8.len(delimiter) > 0) then
newidx = __lua_lib_luautf8_Utf8.find(self, delimiter, idx, true);
else
if (idx >= __lua_lib_luautf8_Utf8.len(self)) then
newidx = nil;
else
newidx = idx + 1;
end;
end;
if (newidx ~= nil) then
local match = __lua_lib_luautf8_Utf8.sub(self, idx, newidx - 1);
ret:push(match);
idx = newidx + __lua_lib_luautf8_Utf8.len(delimiter);
else
ret:push(__lua_lib_luautf8_Utf8.sub(self, idx, __lua_lib_luautf8_Utf8.len(self)));
idx = nil;
end;
end;
do return ret end
end
String.prototype.toString = function(self)
do return self end
end
String.prototype.substring = function(self,startIndex,endIndex)
if (endIndex == nil) then
endIndex = __lua_lib_luautf8_Utf8.len(self);
end;
if (endIndex < 0) then
endIndex = 0;
end;
if (startIndex < 0) then
startIndex = 0;
end;
if (endIndex < startIndex) then
do return __lua_lib_luautf8_Utf8.sub(self, endIndex + 1, startIndex) end;
else
do return __lua_lib_luautf8_Utf8.sub(self, startIndex + 1, endIndex) end;
end;
end
String.prototype.charAt = function(self,index)
do return __lua_lib_luautf8_Utf8.sub(self, index + 1, index + 1) end
end
String.prototype.charCodeAt = function(self,index)
do return __lua_lib_luautf8_Utf8.byte(self, index + 1) end
end
String.prototype.substr = function(self,pos,len)
if ((len == nil) or (len > (pos + __lua_lib_luautf8_Utf8.len(self)))) then
len = __lua_lib_luautf8_Utf8.len(self);
else
if (len < 0) then
len = __lua_lib_luautf8_Utf8.len(self) + len;
end;
end;
if (pos < 0) then
pos = __lua_lib_luautf8_Utf8.len(self) + pos;
end;
if (pos < 0) then
pos = 0;
end;
do return __lua_lib_luautf8_Utf8.sub(self, pos + 1, pos + len) end
end
Std.new = {}
Std.string = function(s)
do return __lua_Boot.__string_rec(s) end;
end
Std.int = function(x)
if (not Math.isFinite(x) or Math.isNaN(x)) then
do return 0 end;
else
do return _hx_bit_clamp(x) end;
end;
end
__haxe_Log.new = {}
__haxe_Log.formatOutput = function(v,infos)
local str = Std.string(v);
if (infos == nil) then
do return str end;
end;
local pstr = Std.string(Std.string(infos.fileName) .. Std.string(":")) .. Std.string(infos.lineNumber);
if (infos.customParams ~= nil) then
local _g = 0;
local _g1 = infos.customParams;
while (_g < _g1.length) do
local v1 = _g1[_g];
_g = _g + 1;
str = Std.string(str) .. Std.string((Std.string(", ") .. Std.string(Std.string(v1))));
end;
end;
do return Std.string(Std.string(pstr) .. Std.string(": ")) .. Std.string(str) end;
end
__haxe_Log.trace = function(v,infos)
local str = __haxe_Log.formatOutput(v, infos);
_hx_print(str);
end
__lua_Boot.new = {}
__lua_Boot.isArray = function(o)
if (_G.type(o) == "table") then
if ((o.__enum__ == nil) and (_G.getmetatable(o) ~= nil)) then
do return _G.getmetatable(o).__index == Array.prototype end;
else
do return false end;
end;
else
do return false end;
end;
end
__lua_Boot.printEnum = function(o,s)
if (o.length == 2) then
do return o[0] end;
else
local str = Std.string(Std.string(o[0])) .. Std.string("(");
s = Std.string(s) .. Std.string("\t");
local _g = 2;
local _g1 = o.length;
while (_g < _g1) do
_g = _g + 1;
local i = _g - 1;
if (i ~= 2) then
str = Std.string(str) .. Std.string((Std.string(",") .. Std.string(__lua_Boot.__string_rec(o[i], s))));
else
str = Std.string(str) .. Std.string(__lua_Boot.__string_rec(o[i], s));
end;
end;
do return Std.string(str) .. Std.string(")") end;
end;
end
__lua_Boot.printClassRec = function(c,result,s)
if (result == nil) then
result = "";
end;
local f = __lua_Boot.__string_rec;
for k,v in pairs(c) do if result ~= '' then result = result .. ', ' end result = result .. k .. ':' .. f(v, s.. ' ') end;
do return result end;
end
__lua_Boot.__string_rec = function(o,s)
if (s == nil) then
s = "";
end;
if (__lua_lib_luautf8_Utf8.len(s) >= 5) then
do return "<...>" end;
end;
local _g = type(o);
if (_g) == "boolean" then
do return tostring(o) end;
elseif (_g) == "function" then
do return "<function>" end;
elseif (_g) == "nil" then
do return "null" end;
elseif (_g) == "number" then
if (o == _G.math.huge) then
do return "Infinity" end;
else
if (o == -_G.math.huge) then
do return "-Infinity" end;
else
if (o == 0) then
do return "0" end;
else
if (o ~= o) then
do return "NaN" end;
else
do return tostring(o) end;
end;
end;
end;
end;
elseif (_g) == "string" then
do return o end;
elseif (_g) == "table" then
if (o.__enum__ ~= nil) then
do return __lua_Boot.printEnum(o, s) end;
else
if ((_hx_wrap_if_string_field(o,'toString') ~= nil) and not __lua_Boot.isArray(o)) then
do return _hx_wrap_if_string_field(o,'toString')(o) end;
else
if (__lua_Boot.isArray(o)) then
local o2 = o;
if (__lua_lib_luautf8_Utf8.len(s) > 5) then
do return "[...]" end;
else
local _g1 = _hx_tab_array({}, 0);
local _g11 = 0;
while (_g11 < o2.length) do
local i = o2[_g11];
_g11 = _g11 + 1;
_g1:push(__lua_Boot.__string_rec(i, Std.string(s) .. Std.string(1)));
end;
do return Std.string(Std.string("[") .. Std.string(_g1:join(","))) .. Std.string("]") end;
end;
else
if (o.__class__ ~= nil) then
do return Std.string(Std.string("{") .. Std.string(__lua_Boot.printClassRec(o, "", Std.string(s) .. Std.string("\t")))) .. Std.string("}") end;
else
local fields = __lua_Boot.fieldIterator(o);
local buffer = ({});
local first = true;
_G.table.insert(buffer, "{ ");
local f = fields;
while (f:hasNext()) do
local f1 = f:next();
if (first) then
first = false;
else
_G.table.insert(buffer, ", ");
end;
_G.table.insert(buffer, Std.string(Std.string(Std.string("") .. Std.string(Std.string(f1))) .. Std.string(" : ")) .. Std.string(__lua_Boot.__string_rec(o[f1], Std.string(s) .. Std.string("\t"))));
end;
_G.table.insert(buffer, " }");
do return _G.table.concat(buffer, "") end;
end;
end;
end;
end;
elseif (_g) == "thread" then
do return "<thread>" end;
elseif (_g) == "userdata" then
local mt = _G.getmetatable(o);
if ((mt ~= nil) and (mt.__tostring ~= nil)) then
do return _G.tostring(o) end;
else
do return "<userdata>" end;
end;else
_G.error("Unknown Lua type",0); end;
end
__lua_Boot.fieldIterator = function(o)
if (_G.type(o) ~= "table") then
do return _hx_o({__fields__={next=true,hasNext=true},next=function(self)
do return nil end;
end,hasNext=function(self)
do return false end;
end}) end;
end;
local tbl = (function()
local _hx_1
if (o.__fields__ ~= nil) then
_hx_1 = o.__fields__; else
_hx_1 = o; end
return _hx_1
end )();
local cur = _G.pairs(tbl);
local next_valid = function(tbl1,val)
while (__lua_Boot.hiddenFields[val] ~= nil) do
val = cur(tbl1, val);
end;
do return val end;
end;
local cur_val = next_valid(tbl, cur(tbl, nil));
do return _hx_o({__fields__={next=true,hasNext=true},next=function(self)
local ret = cur_val;
cur_val = next_valid(tbl, cur(tbl, cur_val));
do return ret end;
end,hasNext=function(self)
do return cur_val ~= nil end;
end}) end;
end
_hx_bit_clamp = function(v)
if v <= 2147483647 and v >= -2147483648 then
if v > 0 then return _G.math.floor(v)
else return _G.math.ceil(v)
end
end
if v > 2251798999999999 then v = v*2 end;
if (v ~= v or math.abs(v) == _G.math.huge) then return nil end
return _hx_bit.band(v, 2147483647 ) - math.abs(_hx_bit.band(v, 2147483648))
end
-- require this for lua 5.1
pcall(require, 'bit')
if bit then
_hx_bit = bit
else
local _hx_bit_raw = _G.require('bit32')
_hx_bit = setmetatable({}, { __index = _hx_bit_raw });
-- lua 5.2 weirdness
_hx_bit.bnot = function(...) return _hx_bit_clamp(_hx_bit_raw.bnot(...)) end;
_hx_bit.bxor = function(...) return _hx_bit_clamp(_hx_bit_raw.bxor(...)) end;
end
_hx_array_mt.__index = Array.prototype
local _hx_static_init = function()
__lua_Boot.hiddenFields = {__id__=true, hx__closures=true, super=true, prototype=true, __fields__=true, __ifields__=true, __class__=true, __properties__=true}
end
_hx_print = print or (function() end)
_hx_wrap_if_string_field = function(o, fld)
if _G.type(o) == 'string' then
if fld == 'length' then
return _G.string.len(o)
else
return String.prototype[fld]
end
else
return o[fld]
end
end
_hx_static_init();
Main.main()

28
_testdata/Generated/Haxe/main.py Normal file
View File

@ -0,0 +1,28 @@
# Generated by Haxe 4.0.5
# coding: utf-8
import sys
class Main:
__slots__ = ()
@staticmethod
def main():
print("Hello World")
class python_internal_MethodClosure:
__slots__ = ("obj", "func")
def __init__(self,obj,func):
self.obj = obj
self.func = func
def __call__(self,*args):
return self.func(self.obj,*args)
Main.main()

2
_testdata/Generated/inline.css Normal file
View File

@ -0,0 +1,2 @@
.accordion{padding:0;margin:0;position:relative;list-style:none}.accordion>*{position:absolute;overflow:hidden;padding:0;margin:0}.accordion .accordion,.accordion.edge-visible,.accordion>*{-webkit-transition:.3s ease all;-moz-transition:.3s ease all;-o-transition:.3s ease all;transition:.3s ease all}.accordion,.accordion>*{will-change:height,transform;-webkit-perspective:90em;-moz-perspective:90em;perspective:90em;-webkit-backface-visibility:hidden;-moz-backface-visibility:hidden;backface-visibility:hidden;-webkit-transform:translate3d(0,0,0);-moz-transform:translate3d(0,0,0);-ms-transform:translateY(0);-o-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.snap.accordion .accordion,.snap.accordion>*{-webkit-transition:none!important;-moz-transition:none!important;-o-transition:none!important;transition:none!important}.accordion>*>:first-child{cursor:pointer;margin:0;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.accordion>*>:last-child{overflow:hidden;-webkit-transition:.3s ease height,.3s step-start visibility;-moz-transition:.3s ease height,.3s step-start visibility;-o-transition:.3s ease height,.3s step-start visibility;transition:.3s ease height,.3s step-start visibility}.accordion>.closed .accordion>.open>:last-child,.accordion>.closed>:last-child{-webkit-transition-timing-function:ease,step-end;-moz-transition-timing-function:ease,step-end;-o-transition-timing-function:ease,step-end;transition-timing-function:ease,step-end;visibility:hidden}
/*# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImFjY29yZGlvbi5jc3MiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQ0EsV0FDQyxRQUFTLEVBQ1QsT0FBUyxFQUNULFNBQVUsU0FDVixXQUFZLEtBRVosYUFDQyxTQUFVLFNBQ1YsU0FBVSxPQUNWLFFBQVMsRUFDVCxPQUFTLEVBSVQsc0JBREEsd0JBREEsYUFHQyxtQkFBb0IsSUFBSSxLQUFLLElBQzdCLGdCQUFvQixJQUFJLEtBQUssSUFDN0IsY0FBb0IsSUFBSSxLQUFLLElBQzdCLFdBQW9CLElBQUksS0FBSyxJQUk5QixXQUNBLGFBQ0MsWUFBYSxNQUFNLENBQUUsVUFDckIsb0JBQXFCLEtBQ3JCLGlCQUFxQixLQUNyQixZQUFxQixLQUVyQiw0QkFBNkIsT0FDN0IseUJBQTZCLE9BQzdCLG9CQUE2QixPQUU3QixrQkFBb0IsbUJBQ3BCLGVBQW9CLG1CQUNwQixjQUFvQixjQUNwQixhQUFvQixtQkFDcEIsVUFBb0IsbUJBS3JCLDJCQURBLGtCQUVDLG1CQUFvQixlQUNwQixnQkFBb0IsZUFDcEIsY0FBb0IsZUFDcEIsV0FBb0IsZUFJckIsMEJBQ0MsT0FBUSxRQUNSLE9BQVEsRUFFUixvQkFBcUIsS0FDckIsaUJBQXFCLEtBQ3JCLGdCQUFxQixLQUNyQixZQUFxQixLQUl0Qix5QkFDQyxTQUFVLE9BQ1YsbUJBQW9CLElBQUksS0FBSyxNQUFNLENBQUUsSUFBSSxXQUFXLFdBQ3BELGdCQUFvQixJQUFJLEtBQUssTUFBTSxDQUFFLElBQUksV0FBVyxXQUNwRCxjQUFvQixJQUFJLEtBQUssTUFBTSxDQUFFLElBQUksV0FBVyxXQUNwRCxXQUFvQixJQUFJLEtBQUssTUFBTSxDQUFFLElBQUksV0FBVyxXQUdwRCxnREFEQSwrQkFFQyxtQ0FBb0MsSUFBSSxDQUFFLFNBQzFDLGdDQUFvQyxJQUFJLENBQUUsU0FDMUMsOEJBQW9DLElBQUksQ0FBRSxTQUMxQywyQkFBb0MsSUFBSSxDQUFFLFNBQzFDLFdBQVkiLCJzb3VyY2VzQ29udGVudCI6WyIvKiogQ29yZSBzdHlsaW5nICovXG4uYWNjb3JkaW9ue1xuXHRwYWRkaW5nOiAwO1xuXHRtYXJnaW46ICAwO1xuXHRwb3NpdGlvbjogcmVsYXRpdmU7XG5cdGxpc3Qtc3R5bGU6IG5vbmU7XG59XG5cdC5hY2NvcmRpb24gPiAqe1xuXHRcdHBvc2l0aW9uOiBhYnNvbHV0ZTtcblx0XHRvdmVyZmxvdzogaGlkZGVuO1xuXHRcdHBhZGRpbmc6IDA7XG5cdFx0bWFyZ2luOiAgMDtcblx0fVxuXHRcdC5hY2NvcmRpb24gPiAqLFxuXHRcdC5hY2NvcmRpb24uZWRnZS12aXNpYmxlLFxuXHRcdC5hY2NvcmRpb24gLmFjY29yZGlvbntcblx0XHRcdC13ZWJraXQtdHJhbnNpdGlvbjogLjNzIGVhc2UgYWxsO1xuXHRcdFx0LW1vei10cmFuc2l0aW9uOiAgICAuM3MgZWFzZSBhbGw7XG5cdFx0XHQtby10cmFuc2l0aW9uOiAgICAgIC4zcyBlYXNlIGFsbDtcblx0XHRcdHRyYW5zaXRpb246ICAgICAgICAgLjNzIGVhc2UgYWxsO1xuXHRcdH1cblx0XHRcblx0XHQvKiogVHJhbnNmb3JtLXJlbGF0ZWQgKi9cblx0XHQuYWNjb3JkaW9uLFxuXHRcdC5hY2NvcmRpb24gPiAqe1xuXHRcdFx0d2lsbC1jaGFuZ2U6IGhlaWdodCwgdHJhbnNmb3JtO1xuXHRcdFx0LXdlYmtpdC1wZXJzcGVjdGl2ZTogOTBlbTtcblx0XHRcdC1tb3otcGVyc3BlY3RpdmU6ICAgIDkwZW07XG5cdFx0XHRwZXJzcGVjdGl2ZTogICAgICAgICA5MGVtO1xuXHRcdFx0XG5cdFx0XHQtd2Via2l0LWJhY2tmYWNlLXZpc2liaWxpdHk6IGhpZGRlbjtcblx0XHRcdC1tb3otYmFja2ZhY2UtdmlzaWJpbGl0eTogICAgaGlkZGVuO1xuXHRcdFx0YmFja2ZhY2UtdmlzaWJpbGl0eTogICAgICAgICBoaWRkZW47XG5cdFx0XHRcblx0XHRcdC13ZWJraXQtdHJhbnNmb3JtOiAgdHJhbnNsYXRlM2QoMCwwLDApO1xuXHRcdFx0LW1vei10cmFuc2Zvcm06ICAgICB0cmFuc2xhdGUzZCgwLDAsMCk7XG5cdFx0XHQtbXMtdHJhbnNmb3JtOiAgICAgIHRyYW5zbGF0ZVkoMCk7XG5cdFx0XHQtby10cmFuc2Zvcm06ICAgICAgIHRyYW5zbGF0ZTNkKDAsMCwwKTtcblx0XHRcdHRyYW5zZm9ybTogICAgICAgICAgdHJhbnNsYXRlM2QoMCwwLDApO1xuXHRcdH1cblx0XHRcblx0XHQvKiogUnVsZSB0byBkaXNhYmxlIHRyYW5zaXRpb25zIGJldHdlZW4gZ2FwIGNvcnJlY3Rpb25zICovXG5cdFx0LnNuYXAuYWNjb3JkaW9uID4gKixcblx0XHQuc25hcC5hY2NvcmRpb24gLmFjY29yZGlvbntcblx0XHRcdC13ZWJraXQtdHJhbnNpdGlvbjogbm9uZSAhaW1wb3J0YW50O1xuXHRcdFx0LW1vei10cmFuc2l0aW9uOiAgICBub25lICFpbXBvcnRhbnQ7XG5cdFx0XHQtby10cmFuc2l0aW9uOiAgICAgIG5vbmUgIWltcG9ydGFudDtcblx0XHRcdHRyYW5zaXRpb246ICAgICAgICAgbm9uZSAhaW1wb3J0YW50O1xuXHRcdH1cblxuXHRcdC8qKiBIZWFkaW5ncyAqL1xuXHRcdC5hY2NvcmRpb24gPiAqID4gOmZpcnN0LWNoaWxke1xuXHRcdFx0Y3Vyc29yOiBwb2ludGVyO1xuXHRcdFx0bWFyZ2luOiAwO1xuXHRcdFx0XG5cdFx0XHQtd2Via2l0LXVzZXItc2VsZWN0OiBub25lO1xuXHRcdFx0LW1vei11c2VyLXNlbGVjdDogICAgbm9uZTtcblx0XHRcdC1tcy11c2VyLXNlbGVjdDogICAgIG5vbmU7XG5cdFx0XHR1c2VyLXNlbGVjdDogICAgICAgICBub25lO1xuXHRcdH1cblx0XHRcblx0XHQvKiogQ29sbGFwc2libGUgY29udGVudCAqL1xuXHRcdC5hY2NvcmRpb24gPiAqID4gOmxhc3QtY2hpbGR7XG5cdFx0XHRvdmVyZmxvdzogaGlkZGVuO1xuXHRcdFx0LXdlYmtpdC10cmFuc2l0aW9uOiAuM3MgZWFzZSBoZWlnaHQsIC4zcyBzdGVwLXN0YXJ0IHZpc2liaWxpdHk7XG5cdFx0XHQtbW96LXRyYW5zaXRpb246ICAgIC4zcyBlYXNlIGhlaWdodCwgLjNzIHN0ZXAtc3RhcnQgdmlzaWJpbGl0eTtcblx0XHRcdC1vLXRyYW5zaXRpb246ICAgICAgLjNzIGVhc2UgaGVpZ2h0LCAuM3Mgc3RlcC1zdGFydCB2aXNpYmlsaXR5O1xuXHRcdFx0dHJhbnNpdGlvbjogICAgICAgICAuM3MgZWFzZSBoZWlnaHQsIC4zcyBzdGVwLXN0YXJ0IHZpc2liaWxpdHk7XG5cdFx0fVxuXHRcdFx0LmFjY29yZGlvbiA+IC5jbG9zZWQgPiA6bGFzdC1jaGlsZCxcblx0XHRcdC5hY2NvcmRpb24gPiAuY2xvc2VkIC5hY2NvcmRpb24gPiAub3BlbiA+IDpsYXN0LWNoaWxke1xuXHRcdFx0XHQtd2Via2l0LXRyYW5zaXRpb24tdGltaW5nLWZ1bmN0aW9uOiBlYXNlLCBzdGVwLWVuZDtcblx0XHRcdFx0LW1vei10cmFuc2l0aW9uLXRpbWluZy1mdW5jdGlvbjogICAgZWFzZSwgc3RlcC1lbmQ7XG5cdFx0XHRcdC1vLXRyYW5zaXRpb24tdGltaW5nLWZ1bmN0aW9uOiAgICAgIGVhc2UsIHN0ZXAtZW5kO1xuXHRcdFx0XHR0cmFuc2l0aW9uLXRpbWluZy1mdW5jdGlvbjogICAgICAgICBlYXNlLCBzdGVwLWVuZDtcblx0XHRcdFx0dmlzaWJpbGl0eTogaGlkZGVuO1xuXHRcdFx0fVxuIl19 */

13
_testdata/Generated/inline.js Normal file
View File

@ -0,0 +1,13 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
class Foo {
constructor(){
this.name = "Foo";
}
}
exports.Foo = Foo;
//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZXNtLmpzIiwic291cmNlcyI6WyJlc20ubWpzIl0sInNvdXJjZXNDb250ZW50IjpbImV4cG9ydCBjbGFzcyBGb28ge1xuXHRjb25zdHJ1Y3Rvcigpe1xuXHRcdHRoaXMubmFtZSA9IFwiRm9vXCI7XG5cdH1cbn07XG4iXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7OztBQUFPLE1BQU0sR0FBRyxDQUFDO0NBQ2hCLFdBQVcsRUFBRTtFQUNaLElBQUksQ0FBQyxJQUFJLEdBQUcsS0FBSyxDQUFDO0VBQ2xCO0NBQ0Q7Ozs7In0=

2
_testdata/Generated/linked.css Normal file
View File

@ -0,0 +1,2 @@
.accordion{padding:0;margin:0;position:relative;list-style:none}.accordion>*{position:absolute;overflow:hidden;padding:0;margin:0}.accordion .accordion,.accordion.edge-visible,.accordion>*{-webkit-transition:.3s ease all;-moz-transition:.3s ease all;-o-transition:.3s ease all;transition:.3s ease all}.accordion,.accordion>*{will-change:height,transform;-webkit-perspective:90em;-moz-perspective:90em;perspective:90em;-webkit-backface-visibility:hidden;-moz-backface-visibility:hidden;backface-visibility:hidden;-webkit-transform:translate3d(0,0,0);-moz-transform:translate3d(0,0,0);-ms-transform:translateY(0);-o-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.snap.accordion .accordion,.snap.accordion>*{-webkit-transition:none!important;-moz-transition:none!important;-o-transition:none!important;transition:none!important}.accordion>*>:first-child{cursor:pointer;margin:0;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.accordion>*>:last-child{overflow:hidden;-webkit-transition:.3s ease height,.3s step-start visibility;-moz-transition:.3s ease height,.3s step-start visibility;-o-transition:.3s ease height,.3s step-start visibility;transition:.3s ease height,.3s step-start visibility}.accordion>.closed .accordion>.open>:last-child,.accordion>.closed>:last-child{-webkit-transition-timing-function:ease,step-end;-moz-transition-timing-function:ease,step-end;-o-transition-timing-function:ease,step-end;transition-timing-function:ease,step-end;visibility:hidden}
/*# sourceMappingURL=linked.css.map */

1
_testdata/Generated/linked.css.map Normal file
View File

@ -0,0 +1 @@
{"version":3,"sources":["accordion.css"],"names":[],"mappings":"AACA,WACC,QAAS,EACT,OAAS,EACT,SAAU,SACV,WAAY,KAEZ,aACC,SAAU,SACV,SAAU,OACV,QAAS,EACT,OAAS,EAIT,sBADA,wBADA,aAGC,mBAAoB,IAAI,KAAK,IAC7B,gBAAoB,IAAI,KAAK,IAC7B,cAAoB,IAAI,KAAK,IAC7B,WAAoB,IAAI,KAAK,IAI9B,WACA,aACC,YAAa,MAAM,CAAE,UACrB,oBAAqB,KACrB,iBAAqB,KACrB,YAAqB,KAErB,4BAA6B,OAC7B,yBAA6B,OAC7B,oBAA6B,OAE7B,kBAAoB,mBACpB,eAAoB,mBACpB,cAAoB,cACpB,aAAoB,mBACpB,UAAoB,mBAKrB,2BADA,kBAEC,mBAAoB,eACpB,gBAAoB,eACpB,cAAoB,eACpB,WAAoB,eAIrB,0BACC,OAAQ,QACR,OAAQ,EAER,oBAAqB,KACrB,iBAAqB,KACrB,gBAAqB,KACrB,YAAqB,KAItB,yBACC,SAAU,OACV,mBAAoB,IAAI,KAAK,MAAM,CAAE,IAAI,WAAW,WACpD,gBAAoB,IAAI,KAAK,MAAM,CAAE,IAAI,WAAW,WACpD,cAAoB,IAAI,KAAK,MAAM,CAAE,IAAI,WAAW,WACpD,WAAoB,IAAI,KAAK,MAAM,CAAE,IAAI,WAAW,WAGpD,gDADA,+BAEC,mCAAoC,IAAI,CAAE,SAC1C,gCAAoC,IAAI,CAAE,SAC1C,8BAAoC,IAAI,CAAE,SAC1C,2BAAoC,IAAI,CAAE,SAC1C,WAAY"}

7
_testdata/Generated/options_main.inherited.yy Normal file
View File

@ -0,0 +1,7 @@
1.0.0←ed6a955d-5826-4f98-a450-10b414266c27←ed6a955d-5826-4f98-a450-10b414266c27|{
"option_gameguid": "e429c80a-2ae4-494a-bee0-f7ed5e39ec9b"
}←1225f6b0-ac20-43bd-a82e-be73fa0b6f4f|{
"targets": 461609314234257646
}←7b2c4976-1e09-44e5-8256-c527145e03bb|{
"targets": 461609314234257646
}

30
_testdata/Generated/ppport.h Normal file
View File

@ -0,0 +1,30 @@
#if 0
<<'SKIP';
#endif
/*
----------------------------------------------------------------------
ppport.h -- Perl/Pollution/Portability Version 3.36
Automatically created by Devel::PPPort running under perl 5.026002.
Do NOT edit this file directly! -- Edit PPPort_pm.PL and the
includes in parts/inc/ instead.
Use 'perldoc ppport.h' to view the documentation below.
----------------------------------------------------------------------
SKIP
=pod
=head1 NAME
ppport.h - Perl/Pollution/Portability version 3.36
=head1 SYNOPSIS
perl ppport.h [options] [source files]
<< TRUNCATED >>

8
_testdata/HTML/attr-swapped.html Normal file
View File

@ -0,0 +1,8 @@
<!DOCTYPE html>
<html>
<head>
<meta
content = "Org mode"
name = "generator" >
</head>
</html>

12
_testdata/HTML/extra-attr.html Normal file
View File

@ -0,0 +1,12 @@
<!DOCTYPE html>
<html>
<head>
<meta
data-foo="Bar"
http-equiv="Content-Type: text/html; charset=UTF-8"
content = "Org mode"
id="Some wicked id"
wrong-content="whoops"
name = "generator" >
</head>
</html>

6
_testdata/HTML/extra-spaces.html Normal file
View File

@ -0,0 +1,6 @@
<!DOCTYPE html>
<html>
<head>
<meta name = "generator" content = "Org mode" />
</head>
</html>

7
_testdata/HTML/extra-tags.html Normal file
View File

@ -0,0 +1,7 @@
<!DOCTYPE html>
<html>
<meta >
<meta name="generator" content="something invalid">
<meta name="generator" content="makeinfo 4.8"
/>
</html>

21
_testdata/HTML/grohtml.html Normal file
View File

@ -0,0 +1,21 @@
<!-- Creator : groff version 1.22.4 -->
<!-- CreationDate: Tue Jul 2 20:06:41 2019 -->
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta name="generator" content="groff -Thtml, see www.gnu.org">
<meta http-equiv="Content-Type" content="text/html; charset=US-ASCII">
<meta name="Content-Style" content="text/css">
<style type="text/css">
p { margin-top: 0; margin-bottom: 0; vertical-align: top }
pre { margin-top: 0; margin-bottom: 0; vertical-align: top }
table { margin-top: 0; margin-bottom: 0; vertical-align: top }
h1 { text-align: center }
</style>
<title></title>
</head>
<body>
... document truncated
</body>
</html>

25
_testdata/HTML/grohtml.xhtml Normal file
View File

@ -0,0 +1,25 @@
<?xml version="1.0" encoding="us-ascii"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1 plus MathML 2.0//EN"
"http://www.w3.org/TR/MathML2/dtd/xhtml-math11-f.dtd"
[<!ENTITY mathml "http://www.w3.org/1998/Math/MathML">]>
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
<head>
<meta name="generator" content="groff -Txhtml, see www.gnu.org"/>
<meta http-equiv="Content-Type" content="text/html; charset=US-ASCII"/>
<meta name="Content-Style" content="text/css"/>
<style type="text/css">
.center { text-align: center }
.right { text-align: right }
p { margin-top: 0; margin-bottom: 0; vertical-align: top }
pre { margin-top: 0; margin-bottom: 0; vertical-align: top }
table { margin-top: 0; margin-bottom: 0; vertical-align: top }
h1 { text-align: center }
</style>
<!-- Creator : groff version 1.22.4 -->
<!-- CreationDate: Tue Jul 2 20:08:09 2019 -->
<title></title>
</head>
<body>
... truncated
</body>
</html>

47
_testdata/HTML/makeinfo.html Normal file
View File

@ -0,0 +1,47 @@
<html lang="en">
<head>
<title>Asm Mode - GNU Emacs Manual</title>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="description" content="GNU Emacs Manual">
<meta name="generator" content="makeinfo 4.8">
<link title="Top" rel="start" href="index.html#Top">
<link rel="up" href="Programs.html#Programs" title="Programs">
<link rel="prev" href="C-Modes.html#C-Modes" title="C Modes">
<link rel="next" href="Fortran.html#Fortran" title="Fortran">
<link href="http://www.gnu.org/software/texinfo/" rel="generator-home" title="Texinfo Homepage">
<!--
This is the `GNU Emacs Manual',
updated for Emacs version {No value for `EMACSVER'}.
Copyright (C) 1985--1987, 1993--2019 Free Software Foundation, Inc.
Permission is granted to copy, distribute and/or modify this
document under the terms of the GNU Free Documentation License,
Version 1.3 or any later version published by the Free Software
Foundation; with the Invariant Sections being ``The GNU
Manifesto,'' ``Distribution'' and ``GNU GENERAL PUBLIC LICENSE,''
with the Front-Cover Texts being ``A GNU Manual,'' and with the
Back-Cover Texts as in (a) below. A copy of the license is
included in the section entitled ``GNU Free Documentation
License.''
(a) The FSF's Back-Cover Text is: ``You have the freedom to copy
and modify this GNU manual. Buying copies from the FSF supports
it in developing GNU and promoting software freedom.''
-->
<meta http-equiv="Content-Style-Type" content="text/css">
<style type="text/css"><!--
pre.display { font-family:inherit }
pre.format { font-family:inherit }
pre.smalldisplay { font-family:inherit; font-size:smaller }
pre.smallformat { font-family:inherit; font-size:smaller }
pre.smallexample { font-size:smaller }
pre.smalllisp { font-size:smaller }
span.sc { font-variant:small-caps }
span.roman { font-family:serif; font-weight:normal; }
span.sansserif { font-family:sans-serif; font-weight:normal; }
--></style>
</head>
<body>
</body>
</html>

40
_testdata/HTML/mandoc.html Normal file
View File

@ -0,0 +1,40 @@
<!DOCTYPE html>
<html>
<!-- This is an automatically generated file. Do not edit.
$OpenBSD: mandoc.1,v 1.161 2019/02/23 18:52:45 schwarze Exp $
Copyright (c) 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
Copyright (c) 2012, 2014-2018 Ingo Schwarze <schwarze@openbsd.org>
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-->
<head>
<meta charset="utf-8"/>
<style>
table.head, table.foot { width: 100%; }
td.head-rtitle, td.foot-os { text-align: right; }
td.head-vol { text-align: center; }
div.Pp { margin: 1ex 0ex; }
div.Nd, div.Bf, div.Op { display: inline; }
span.Pa, span.Ad { font-style: italic; }
span.Ms { font-weight: bold; }
dl.Bl-diag > dt { font-weight: bold; }
code.Nm, code.Fl, code.Cm, code.Ic, code.In, code.Fd, code.Fn,
code.Cd { font-weight: bold; font-family: inherit; }
</style>
<title>MANDOC(1)</title>
</head>
<body>
... document truncated
</body>
</html>

4
_testdata/HTML/no-content.html Normal file
View File

@ -0,0 +1,4 @@
<!DOCTYPE html>
<html>
<meta name="generator" value="Some sick tool nobody's using yet" />
</html>

73
_testdata/HTML/node78.html Normal file
View File

@ -0,0 +1,73 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
<!--Converted with jLaTeX2HTML 2002-2-1 (1.70) JA patch-2.0
patched version by: Kenshi Muto, Debian Project.
* modified by: Shige TAKENO
LaTeX2HTML 2002-2-1 (1.70),
original version by: Nikos Drakos, CBLU, University of Leeds
* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
* with significant contributions from:
Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
<HTML>
<HEAD>
<TITLE>Quality/Complexity metric plugins</TITLE>
<META NAME="description" CONTENT="Quality/Complexity metric plugins">
<META NAME="keywords" CONTENT="main">
<META NAME="resource-type" CONTENT="document">
<META NAME="distribution" CONTENT="global">
<META NAME="Generator" CONTENT="jLaTeX2HTML v2002-2-1 JA patch-2.0">
<META HTTP-EQUIV="Content-Style-Type" CONTENT="text/css">
<LINK REL="STYLESHEET" HREF="main.css">
<LINK REL="next" HREF="node79.html">
<LINK REL="previous" HREF="node77.html">
<LINK REL="up" HREF="node74.html">
<LINK REL="next" HREF="node79.html">
</HEAD>
<BODY >
<!--Navigation Panel-->
<A NAME="tex2html1237"
HREF="node79.html">
<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
SRC="file:/usr/share/latex2html/icons/next.png"></A>
<A NAME="tex2html1233"
HREF="node74.html">
<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
SRC="file:/usr/share/latex2html/icons/up.png"></A>
<A NAME="tex2html1227"
HREF="node77.html">
<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
SRC="file:/usr/share/latex2html/icons/prev.png"></A>
<A NAME="tex2html1235"
HREF="node1.html">
<IMG WIDTH="65" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="contents"
SRC="file:/usr/share/latex2html/icons/contents.png"></A>
<BR>
<B> Next:</B> <A NAME="tex2html1238"
HREF="node79.html">Time signal generator</A>
<B> Up:</B> <A NAME="tex2html1234"
HREF="node74.html">Additional simulator components</A>
<B> Previous:</B> <A NAME="tex2html1228"
HREF="node77.html">Weather scenario ()</A>
&nbsp; <B> <A NAME="tex2html1236"
HREF="node1.html">Contents</A></B>
<BR>
<BR>
<!--End of Navigation Panel-->
<H3><A NAME="SECTION000101400000000000000">
Quality/Complexity metric plugins</A>
</H3>
Various plugins to measure schedule quality and problem complexity to suit the application.
<P>
<BR><HR>
<ADDRESS>
Steve Fraser
2008-01-31
</ADDRESS>
</BODY>
</HTML>

20
_testdata/HTML/org-mode.html Normal file
View File

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
<head>
<!-- 2019-06-08 Sat 04:15 -->
<meta http-equiv="Content-Type" /
content="text/html;charset=utf-8" />
<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>unpackaged.el</title>
<meta name="generator" content="Org mode" />
<meta name="author" content="Adam Porter" />
<style type="text/css">
<!--/*--><![CDATA[/*><!--*/
</style>
</head>
<body>
</body>
</html>

31
_testdata/HTML/pages.html Normal file
View File

@ -0,0 +1,31 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<title>Related Pages</title>
<link href="qt.css" rel="stylesheet" type="text/css"/>
</head>
<body>
<div class=header>
<a class=headerLink href="index.html">Main Page</a> &middot;
<a class=headerLink href="classoverview.html">Class Overview</a> &middot;
<a class=headerLink href="hierarchy.html">Hierarchy</a> &middot;
<a class=headerLink href="annotated.html">All Classes</a>
</div>
<!-- Generated by Doxygen 1.8.1.2 -->
</div><!-- top -->
<div class="header">
<div class="headertitle">
<div class="title">Related Pages</div> </div>
</div><!--header-->
<div class="contents">
<div class="textblock">Here is a list of all related documentation pages:</div><div class="directory">
<table class="directory">
<tr id="row_0_" class="even"><td class="entry"><img src="ftv2node.png" alt="o" width="16" height="22" /><a class="el" href="classoverview.html" target="_self">Class Overview</a></td><td class="desc"></td></tr>
<tr id="row_1_"><td class="entry"><img src="ftv2lastnode.png" alt="\" width="16" height="22" /><a class="el" href="thelayoutsystem.html" target="_self">The Layout System</a></td><td class="desc"></td></tr>
</table>
</div><!-- directory -->
</div><!-- contents -->
<div class="footer" />Generated with <a href="http://www.doxygen.org/index.html">Doxygen</a> 1.8.1.2</div>
</body>
</html>

10
_testdata/HTML/quotes-double.html Normal file
View File

@ -0,0 +1,10 @@
<!DOCTYPE html>
<html>
<head>
<meta
name ="generator"
content =
"Org mode"
>
</head>
</html>

6
_testdata/HTML/quotes-none.html Normal file
View File

@ -0,0 +1,6 @@
<!DOCTYPE html>
<html>
<head>
<meta name = generator content = makeinfo />
</head>
</html>

6
_testdata/HTML/quotes-single.html Normal file
View File

@ -0,0 +1,6 @@
<!DOCTYPE html>
<html>
<head>
<meta name = 'generator' content = 'Org mode'>
</head>
</html>

4
_testdata/HTML/ronn.html Normal file
View File

@ -0,0 +1,4 @@
<!DOCTYPE html>
<html>
<meta name="generator" value="Ronn/v0.7.3 (http://github.com/rtomayko/ronn/tree/0.7.3)" />
</html>

4
_testdata/HTML/unknown.html Normal file
View File

@ -0,0 +1,4 @@
<!DOCTYPE html>
<html>
<meta name="generator" content="Some sick tool nobody's using yet" />
</html>

8
_testdata/HTML/uppercase.html Normal file
View File

@ -0,0 +1,8 @@
<!DOCTYPE html>
<html>
<head>
<META NAME= 'GENERATOR'
CONTENT
= 'ORG MODE'>
</head>
</html>

15
_testdata/JSON/2ea73365-b6f1-4bd1-a454-d57a67e50684.yy Normal file
View File

@ -0,0 +1,15 @@
{
"id": "2ea73365-b6f1-4bd1-a454-d57a67e50684",
"modelName": "GMFolder",
"mvc": "1.1",
"name": "2ea73365-b6f1-4bd1-a454-d57a67e50684",
"children": [
"d74cdac8-2717-46a5-a8ed-5f732e02a268",
"b68cebe1-d0fa-44ed-8a26-595c2885a1fc",
"1e95eeaf-b11a-44a2-a1b0-bd201f6366ee"
],
"filterType": "GMSprite",
"folderName": "sprites",
"isDefaultView": false,
"localisedFolderName": "ResourceTree_Sprites"
}

374
_testdata/JSON/GMS2_Project.yyp Normal file
View File

@ -0,0 +1,374 @@
{
"id": "62aa8308-837c-4613-a44e-a1306eee5c3c",
"modelName": "GMProject",
"mvc": "1.0",
"IsDnDProject": false,
"configs": [
],
"option_ecma": false,
"parentProject": {
"id": "18731a27-007b-4c7c-802b-b7eac446e27a",
"modelName": "GMProjectParent",
"mvc": "1.0",
"alteredResources": [
{
"Key": "ed6a955d-5826-4f98-a450-10b414266c27",
"Value": {
"configDeltas": [
"inherited"
],
"id": "a3c2778c-ba48-40f4-b180-c31dd0bf91c2",
"resourcePath": "options\\main\\options_main.yy",
"resourceType": "GMMainOptions"
}
}
],
"hiddenResources": [
],
"projectPath": "${base_project}"
},
"resources": [
{
"Key": "05f45e8d-a727-4650-b0c2-0ac5f793f164",
"Value": {
"id": "6f75d81a-3fe8-484f-8178-ce66c6b9b8cf",
"resourcePath": "views\\05f45e8d-a727-4650-b0c2-0ac5f793f164.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "128687b6-cf4b-4bad-bfec-52960e01a538",
"Value": {
"id": "ec6296c1-a5d9-4c21-b3e3-5d4363a2070c",
"resourcePath": "views\\128687b6-cf4b-4bad-bfec-52960e01a538.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "1e95eeaf-b11a-44a2-a1b0-bd201f6366ee",
"Value": {
"id": "b109c79d-b9d0-4d6a-8b13-e460efb93824",
"resourcePath": "views\\1e95eeaf-b11a-44a2-a1b0-bd201f6366ee.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "227a1abf-7147-4a8b-bb31-62a5d3a9a07e",
"Value": {
"id": "d2010bb7-ac3c-4717-a93e-edb7e447e257",
"resourcePath": "objects\\obj_king\\obj_king.yy",
"resourceType": "GMObject"
}
},
{
"Key": "297651c9-b7a7-4a65-97f1-8ceb3b64f65d",
"Value": {
"id": "ab37c354-5987-4384-ae57-4edb8c73110d",
"resourcePath": "views\\297651c9-b7a7-4a65-97f1-8ceb3b64f65d.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "2d0a83ba-80a2-40b2-8805-93347555ef97",
"Value": {
"id": "cb703663-2592-4c26-bfbb-a11a76390b27",
"resourcePath": "objects\\par_depthobject\\par_depthobject.yy",
"resourceType": "GMObject"
}
},
{
"Key": "2ea73365-b6f1-4bd1-a454-d57a67e50684",
"Value": {
"id": "1136167c-dc54-4c46-b98d-c616e004866b",
"resourcePath": "views\\2ea73365-b6f1-4bd1-a454-d57a67e50684.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "31ff7d32-f0cd-4e2e-a129-338e3e580a1a",
"Value": {
"id": "027854a7-69ad-4dd9-bb97-94cf1cd09c90",
"resourcePath": "sprites\\spr_shadow\\spr_shadow.yy",
"resourceType": "GMSprite"
}
},
{
"Key": "48b46420-2f90-4794-9064-b6fdfad99fbd",
"Value": {
"id": "843e68a6-9d39-4f4c-99b0-7e2beb280bdc",
"resourcePath": "views\\48b46420-2f90-4794-9064-b6fdfad99fbd.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "4c4a5f6b-851e-45f2-9d16-4e770693d0d6",
"Value": {
"id": "d27ca460-a737-425d-b658-fda4e232da2d",
"resourcePath": "views\\4c4a5f6b-851e-45f2-9d16-4e770693d0d6.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "53f34530-342f-440f-8cd3-3df3c5bfc232",
"Value": {
"id": "7189b8af-fafa-4643-b05a-1efec553c075",
"resourcePath": "views\\53f34530-342f-440f-8cd3-3df3c5bfc232.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "556893b0-c414-45f4-b592-5ae7021b4e74",
"Value": {
"id": "aec12608-534e-4e2c-ad54-c3996c929309",
"resourcePath": "objects\\depthsorter\\depthsorter.yy",
"resourceType": "GMObject"
}
},
{
"Key": "583b7bc3-11a0-419c-8028-17f69745ce53",
"Value": {
"id": "f6fd0fc5-a491-4c44-b798-e69b3ff043f7",
"resourcePath": "sprites\\spr_knight\\spr_knight.yy",
"resourceType": "GMSprite"
}
},
{
"Key": "5fd06095-db49-460e-9954-34d8113518c0",
"Value": {
"id": "043b6ae1-c67b-4478-8d4a-98c54da49c59",
"resourcePath": "views\\5fd06095-db49-460e-9954-34d8113518c0.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "62e5c66d-8c36-4bc8-af71-c4119d077b56",
"Value": {
"id": "57e7a582-a710-4c41-9549-aa504dc9b2b0",
"resourcePath": "sprites\\spr_player_idle\\spr_player_idle.yy",
"resourceType": "GMSprite"
}
},
{
"Key": "638c7030-5359-4b3c-8e90-60cb3fcb9704",
"Value": {
"id": "60e7c880-7135-4f49-adb6-92520a22ca65",
"resourcePath": "objects\\obj_greentree\\obj_greentree.yy",
"resourceType": "GMObject"
}
},
{
"Key": "78fc92d3-2675-43e1-a794-30066b3aea37",
"Value": {
"id": "7b9e43bd-7dbc-4b55-b3a5-88176788511a",
"resourcePath": "objects\\obj_knight\\obj_knight.yy",
"resourceType": "GMObject"
}
},
{
"Key": "82f217a6-9373-47d2-85d9-14ce23a512a0",
"Value": {
"id": "bf7cdb17-ee68-4006-b102-afa68538dd7e",
"resourcePath": "views\\82f217a6-9373-47d2-85d9-14ce23a512a0.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "83d63ce4-d03f-444b-b478-522ad5b314a9",
"Value": {
"id": "a927ec72-d381-4816-9d70-c9b78f3e9eb0",
"resourcePath": "views\\83d63ce4-d03f-444b-b478-522ad5b314a9.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "8a96eadc-2360-466a-ae4a-7a71b1cfda17",
"Value": {
"id": "f5608dcf-9a81-490d-9373-fa59fc44fdbf",
"resourcePath": "tilesets\\tileset_1\\tileset_1.yy",
"resourceType": "GMTileSet"
}
},
{
"Key": "8df8cbdc-c273-479a-83c9-496f2e62d294",
"Value": {
"id": "a74e5eb9-978f-47f3-aa50-20b7a3fa9baa",
"resourcePath": "views\\8df8cbdc-c273-479a-83c9-496f2e62d294.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "9c35e2b5-ef9d-468c-8d88-8e8636785ca0",
"Value": {
"id": "fe440907-a939-4974-83aa-853900a43472",
"resourcePath": "views\\9c35e2b5-ef9d-468c-8d88-8e8636785ca0.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "a09f41cd-a4f3-4818-9942-6c258e8a0b0e",
"Value": {
"id": "ae291c5c-f27e-44a2-a6a7-8192ba7a8297",
"resourcePath": "views\\a09f41cd-a4f3-4818-9942-6c258e8a0b0e.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "a9188620-a624-4a5a-83ae-a1b53faf038b",
"Value": {
"id": "3a6a5b33-9543-4fad-922d-403ffb3e8fa2",
"resourcePath": "options\\linux\\options_linux.yy",
"resourceType": "GMLinuxOptions"
}
},
{
"Key": "a9c4bc40-93a0-4449-a951-9bfc82428101",
"Value": {
"id": "6b8e239a-9414-498c-85ae-23d5f8c1cb90",
"resourcePath": "objects\\obj_collision\\obj_collision.yy",
"resourceType": "GMObject"
}
},
{
"Key": "aa4ccde8-d128-4dfe-a243-5efe2da9ff0d",
"Value": {
"id": "41ccb1a0-cf8e-4b28-a1fb-ce53f209bcd0",
"resourcePath": "views\\aa4ccde8-d128-4dfe-a243-5efe2da9ff0d.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "ac913661-4852-4ce1-89ce-3a031aed6119",
"Value": {
"id": "09ddc355-efe1-4b77-9431-3bc87009e880",
"resourcePath": "sprites\\spr_tileset1\\spr_tileset1.yy",
"resourceType": "GMSprite"
}
},
{
"Key": "acfe2407-bee7-4ce9-ac58-d3327085f21b",
"Value": {
"id": "201500e6-154c-4c05-8221-57fdd65f74e7",
"resourcePath": "sprites\\spr_player\\spr_player.yy",
"resourceType": "GMSprite"
}
},
{
"Key": "af01d96e-a301-41cc-9c82-eb60af19cbae",
"Value": {
"id": "34f0d902-d0aa-4ab8-8d02-9d91150b816c",
"resourcePath": "views\\af01d96e-a301-41cc-9c82-eb60af19cbae.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "b4fe38c4-c338-4d32-b85f-1b41d9a2446d",
"Value": {
"id": "ae19ee11-6b01-4f2b-8f55-adf5cb081961",
"resourcePath": "sprites\\spr_collision\\spr_collision.yy",
"resourceType": "GMSprite"
}
},
{
"Key": "b68cebe1-d0fa-44ed-8a26-595c2885a1fc",
"Value": {
"id": "1f6a859b-68fa-4564-bcb6-1fe387319eb4",
"resourcePath": "views\\b68cebe1-d0fa-44ed-8a26-595c2885a1fc.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "bd8b3db8-e5e6-44f1-bfcf-45d3242a0122",
"Value": {
"id": "4450def6-f270-4dd7-8896-c9d8ac34be9b",
"resourcePath": "sprites\\spr_scout\\spr_scout.yy",
"resourceType": "GMSprite"
}
},
{
"Key": "bffbcd96-a8c7-431f-af4f-7f66e8f84189",
"Value": {
"id": "bfee4d13-6e16-41f6-9fb7-095126af6fcd",
"resourcePath": "rooms\\rm_1\\rm_1.yy",
"resourceType": "GMRoom"
}
},
{
"Key": "cc3b80a7-5d09-4bea-b9c8-b093f1b244c8",
"Value": {
"id": "ac219fd8-ba00-4bff-857c-93d7cdb409c8",
"resourcePath": "views\\cc3b80a7-5d09-4bea-b9c8-b093f1b244c8.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "cc98d028-7bdd-4680-85f3-c87a7baa481e",
"Value": {
"id": "a84e7ce1-a928-4a86-bd79-8355f266e12e",
"resourcePath": "options\\windows\\options_windows.yy",
"resourceType": "GMWindowsOptions"
}
},
{
"Key": "d3e83dba-5be8-436f-866d-9a5a2120adb1",
"Value": {
"id": "ca2f0120-77ec-4bb4-8b0b-c12af28e2db0",
"resourcePath": "views\\d3e83dba-5be8-436f-866d-9a5a2120adb1.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "d74cdac8-2717-46a5-a8ed-5f732e02a268",
"Value": {
"id": "3b26d2bb-6a56-4a30-9c5a-47c3bcc83bb9",
"resourcePath": "views\\d74cdac8-2717-46a5-a8ed-5f732e02a268.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "e3e41b8b-b981-42ce-aab7-e6265764c7a8",
"Value": {
"id": "a24e4fd7-f4da-4da0-aff1-da1bb864c0b4",
"resourcePath": "views\\e3e41b8b-b981-42ce-aab7-e6265764c7a8.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "f1fbb653-bca3-4725-9e81-6a3556760d0c",
"Value": {
"id": "76dcd643-64fe-49d6-83ff-e15184b56150",
"resourcePath": "objects\\obj_scout\\obj_scout.yy",
"resourceType": "GMObject"
}
},
{
"Key": "f418569b-3bdd-4706-a0e4-364317f54032",
"Value": {
"id": "1292899d-f280-4f75-93b4-b2ad6099dd3d",
"resourcePath": "options\\mac\\options_mac.yy",
"resourceType": "GMMacOptions"
}
},
{
"Key": "f48fa589-53e5-4ce0-a010-c16b38192e31",
"Value": {
"id": "e950beb0-3503-4c3d-9cb9-8743c91d7c94",
"resourcePath": "views\\f48fa589-53e5-4ce0-a010-c16b38192e31.yy",
"resourceType": "GMFolder"
}
},
{
"Key": "fd46d5b1-fe49-4ecf-b809-0a2449808fbe",
"Value": {
"id": "09d46108-701a-4592-906f-bcb479008507",
"resourcePath": "sprites\\spr_tree\\spr_tree.yy",
"resourceType": "GMSprite"
}
}
],
"script_order": [
],
"tutorial": ""
}

625
_testdata/Java/GrammarKit.java Normal file
View File

@ -0,0 +1,625 @@
// This is a generated file. Not intended for manual editing.
package org.intellij.grammar.parser;
import com.intellij.lang.PsiBuilder;
import com.intellij.lang.PsiBuilder.Marker;
import static org.intellij.grammar.psi.BnfTypes.*;
import static org.intellij.grammar.parser.GeneratedParserUtilBase.*;
import com.intellij.psi.tree.IElementType;
import com.intellij.lang.ASTNode;
import com.intellij.psi.tree.TokenSet;
import com.intellij.lang.PsiParser;
import com.intellij.lang.LightPsiParser;
@SuppressWarnings({"SimplifiableIfStatement", "UnusedAssignment"})
public class GrammarParser implements PsiParser, LightPsiParser {
public ASTNode parse(IElementType t, PsiBuilder b) {
parseLight(t, b);
return b.getTreeBuilt();
}
public void parseLight(IElementType t, PsiBuilder b) {
boolean r;
b = adapt_builder_(t, b, this, EXTENDS_SETS_);
Marker m = enter_section_(b, 0, _COLLAPSE_, null);
if (t == BNF_ATTR) {
r = attr(b, 0);
}
else if (t == BNF_ATTR_PATTERN) {
r = attr_pattern(b, 0);
}
else if (t == BNF_ATTR_VALUE) {
r = attr_value(b, 0);
}
else if (t == BNF_ATTRS) {
r = attrs(b, 0);
}
else if (t == BNF_CHOICE) {
r = choice(b, 0);
}
else if (t == BNF_EXPRESSION) {
r = expression(b, 0);
}
else if (t == BNF_LITERAL_EXPRESSION) {
r = literal_expression(b, 0);
}
else if (t == BNF_MODIFIER) {
r = modifier(b, 0);
}
else if (t == BNF_PAREN_EXPRESSION) {
r = paren_expression(b, 0);
}
else if (t == BNF_PREDICATE) {
r = predicate(b, 0);
}
else if (t == BNF_PREDICATE_SIGN) {
r = predicate_sign(b, 0);
}
else if (t == BNF_QUANTIFIED) {
r = quantified(b, 0);
}
else if (t == BNF_QUANTIFIER) {
r = quantifier(b, 0);
}
else if (t == BNF_REFERENCE_OR_TOKEN) {
r = reference_or_token(b, 0);
}
else if (t == BNF_RULE) {
r = rule(b, 0);
}
else if (t == BNF_SEQUENCE) {
r = sequence(b, 0);
}
else if (t == BNF_STRING_LITERAL_EXPRESSION) {
r = string_literal_expression(b, 0);
}
else {
r = parse_root_(t, b, 0);
}
exit_section_(b, 0, m, t, r, true, TRUE_CONDITION);
}
protected boolean parse_root_(IElementType t, PsiBuilder b, int l) {
return grammar(b, l + 1);
}
public static final TokenSet[] EXTENDS_SETS_ = new TokenSet[] {
create_token_set_(BNF_LITERAL_EXPRESSION, BNF_STRING_LITERAL_EXPRESSION),
create_token_set_(BNF_CHOICE, BNF_EXPRESSION, BNF_LITERAL_EXPRESSION, BNF_PAREN_EXPRESSION,
BNF_PREDICATE, BNF_QUANTIFIED, BNF_REFERENCE_OR_TOKEN, BNF_SEQUENCE,
BNF_STRING_LITERAL_EXPRESSION),
};
/* ********************************************************** */
// id attr_pattern? '=' attr_value ';'?
public static boolean attr(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "attr")) return false;
boolean r, p;
Marker m = enter_section_(b, l, _NONE_, "<attr>");
r = consumeToken(b, BNF_ID);
p = r; // pin = 1
r = r && report_error_(b, attr_1(b, l + 1));
r = p && report_error_(b, consumeToken(b, BNF_OP_EQ)) && r;
r = p && report_error_(b, attr_value(b, l + 1)) && r;
r = p && attr_4(b, l + 1) && r;
exit_section_(b, l, m, BNF_ATTR, r, p, attr_recover_until_parser_);
return r || p;
}
// attr_pattern?
private static boolean attr_1(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "attr_1")) return false;
attr_pattern(b, l + 1);
return true;
}
// ';'?
private static boolean attr_4(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "attr_4")) return false;
consumeToken(b, BNF_SEMICOLON);
return true;
}
/* ********************************************************** */
// '(' string ')'
public static boolean attr_pattern(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "attr_pattern")) return false;
if (!nextTokenIs(b, BNF_LEFT_PAREN)) return false;
boolean r;
Marker m = enter_section_(b);
r = consumeToken(b, BNF_LEFT_PAREN);
r = r && consumeToken(b, BNF_STRING);
r = r && consumeToken(b, BNF_RIGHT_PAREN);
exit_section_(b, m, BNF_ATTR_PATTERN, r);
return r;
}
/* ********************************************************** */
// !'}'
static boolean attr_recover_until(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "attr_recover_until")) return false;
boolean r;
Marker m = enter_section_(b, l, _NOT_, null);
r = !consumeToken(b, BNF_RIGHT_BRACE);
exit_section_(b, l, m, null, r, false, null);
return r;
}
/* ********************************************************** */
// (reference_or_token | literal_expression) !'='
public static boolean attr_value(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "attr_value")) return false;
boolean r;
Marker m = enter_section_(b, l, _NONE_, "<attr value>");
r = attr_value_0(b, l + 1);
r = r && attr_value_1(b, l + 1);
exit_section_(b, l, m, BNF_ATTR_VALUE, r, false, null);
return r;
}
// reference_or_token | literal_expression
private static boolean attr_value_0(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "attr_value_0")) return false;
boolean r;
Marker m = enter_section_(b);
r = reference_or_token(b, l + 1);
if (!r) r = literal_expression(b, l + 1);
exit_section_(b, m, null, r);
return r;
}
// !'='
private static boolean attr_value_1(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "attr_value_1")) return false;
boolean r;
Marker m = enter_section_(b, l, _NOT_, null);
r = !consumeToken(b, BNF_OP_EQ);
exit_section_(b, l, m, null, r, false, null);
return r;
}
/* ********************************************************** */
// '{' attr* '}'
public static boolean attrs(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "attrs")) return false;
if (!nextTokenIs(b, BNF_LEFT_BRACE)) return false;
boolean r, p;
Marker m = enter_section_(b, l, _NONE_, null);
r = consumeToken(b, BNF_LEFT_BRACE);
p = r; // pin = 1
r = r && report_error_(b, attrs_1(b, l + 1));
r = p && consumeToken(b, BNF_RIGHT_BRACE) && r;
exit_section_(b, l, m, BNF_ATTRS, r, p, null);
return r || p;
}
// attr*
private static boolean attrs_1(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "attrs_1")) return false;
int c = current_position_(b);
while (true) {
if (!attr(b, l + 1)) break;
if (!empty_element_parsed_guard_(b, "attrs_1", c)) break;
c = current_position_(b);
}
return true;
}
/* ********************************************************** */
// '{' sequence ('|' sequence)* '}' | sequence choice_tail*
public static boolean choice(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "choice")) return false;
boolean r;
Marker m = enter_section_(b, l, _COLLAPSE_, "<choice>");
r = choice_0(b, l + 1);
if (!r) r = choice_1(b, l + 1);
exit_section_(b, l, m, BNF_CHOICE, r, false, null);
return r;
}
// '{' sequence ('|' sequence)* '}'
private static boolean choice_0(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "choice_0")) return false;
boolean r;
Marker m = enter_section_(b);
r = consumeToken(b, BNF_LEFT_BRACE);
r = r && sequence(b, l + 1);
r = r && choice_0_2(b, l + 1);
r = r && consumeToken(b, BNF_RIGHT_BRACE);
exit_section_(b, m, null, r);
return r;
}
// ('|' sequence)*
private static boolean choice_0_2(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "choice_0_2")) return false;
int c = current_position_(b);
while (true) {
if (!choice_0_2_0(b, l + 1)) break;
if (!empty_element_parsed_guard_(b, "choice_0_2", c)) break;
c = current_position_(b);
}
return true;
}
// '|' sequence
private static boolean choice_0_2_0(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "choice_0_2_0")) return false;
boolean r;
Marker m = enter_section_(b);
r = consumeToken(b, BNF_OP_OR);
r = r && sequence(b, l + 1);
exit_section_(b, m, null, r);
return r;
}
// sequence choice_tail*
private static boolean choice_1(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "choice_1")) return false;
boolean r;
Marker m = enter_section_(b);
r = sequence(b, l + 1);
r = r && choice_1_1(b, l + 1);
exit_section_(b, m, null, r);
return r;
}
// choice_tail*
private static boolean choice_1_1(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "choice_1_1")) return false;
int c = current_position_(b);
while (true) {
if (!choice_tail(b, l + 1)) break;
if (!empty_element_parsed_guard_(b, "choice_1_1", c)) break;
c = current_position_(b);
}
return true;
}
/* ********************************************************** */
// '|' sequence
static boolean choice_tail(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "choice_tail")) return false;
if (!nextTokenIs(b, BNF_OP_OR)) return false;
boolean r, p;
Marker m = enter_section_(b, l, _NONE_, null);
r = consumeToken(b, BNF_OP_OR);
p = r; // pin = 1
r = r && sequence(b, l + 1);
exit_section_(b, l, m, null, r, p, null);
return r || p;
}
/* ********************************************************** */
// choice?
public static boolean expression(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "expression")) return false;
Marker m = enter_section_(b, l, _COLLAPSE_, "<expression>");
choice(b, l + 1);
exit_section_(b, l, m, BNF_EXPRESSION, true, false, null);
return true;
}
/* ********************************************************** */
// (attrs | rule) *
static boolean grammar(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "grammar")) return false;
int c = current_position_(b);
while (true) {
if (!grammar_0(b, l + 1)) break;
if (!empty_element_parsed_guard_(b, "grammar", c)) break;
c = current_position_(b);
}
return true;
}
// attrs | rule
private static boolean grammar_0(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "grammar_0")) return false;
boolean r;
Marker m = enter_section_(b);
r = attrs(b, l + 1);
if (!r) r = rule(b, l + 1);
exit_section_(b, m, null, r);
return r;
}
/* ********************************************************** */
// string_literal_expression | number
public static boolean literal_expression(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "literal_expression")) return false;
if (!nextTokenIs(b, "<literal expression>", BNF_NUMBER, BNF_STRING)) return false;
boolean r;
Marker m = enter_section_(b, l, _COLLAPSE_, "<literal expression>");
r = string_literal_expression(b, l + 1);
if (!r) r = consumeToken(b, BNF_NUMBER);
exit_section_(b, l, m, BNF_LITERAL_EXPRESSION, r, false, null);
return r;
}
/* ********************************************************** */
// 'private' | 'external' | 'wrapped'
public static boolean modifier(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "modifier")) return false;
boolean r;
Marker m = enter_section_(b, l, _NONE_, "<modifier>");
r = consumeToken(b, "private");
if (!r) r = consumeToken(b, "external");
if (!r) r = consumeToken(b, "wrapped");
exit_section_(b, l, m, BNF_MODIFIER, r, false, null);
return r;
}
/* ********************************************************** */
// quantified | predicate
static boolean option(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "option")) return false;
boolean r;
Marker m = enter_section_(b);
r = quantified(b, l + 1);
if (!r) r = predicate(b, l + 1);
exit_section_(b, m, null, r);
return r;
}
/* ********************************************************** */
// '(' expression ')'
public static boolean paren_expression(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "paren_expression")) return false;
if (!nextTokenIs(b, BNF_LEFT_PAREN)) return false;
boolean r, p;
Marker m = enter_section_(b, l, _NONE_, null);
r = consumeToken(b, BNF_LEFT_PAREN);
p = r; // pin = 1
r = r && report_error_(b, expression(b, l + 1));
r = p && consumeToken(b, BNF_RIGHT_PAREN) && r;
exit_section_(b, l, m, BNF_PAREN_EXPRESSION, r, p, null);
return r || p;
}
/* ********************************************************** */
// predicate_sign simple
public static boolean predicate(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "predicate")) return false;
if (!nextTokenIs(b, "<predicate>", BNF_OP_NOT, BNF_OP_AND)) return false;
boolean r;
Marker m = enter_section_(b, l, _NONE_, "<predicate>");
r = predicate_sign(b, l + 1);
r = r && simple(b, l + 1);
exit_section_(b, l, m, BNF_PREDICATE, r, false, null);
return r;
}
/* ********************************************************** */
// '&' | '!'
public static boolean predicate_sign(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "predicate_sign")) return false;
if (!nextTokenIs(b, "<predicate sign>", BNF_OP_NOT, BNF_OP_AND)) return false;
boolean r;
Marker m = enter_section_(b, l, _NONE_, "<predicate sign>");
r = consumeToken(b, BNF_OP_AND);
if (!r) r = consumeToken(b, BNF_OP_NOT);
exit_section_(b, l, m, BNF_PREDICATE_SIGN, r, false, null);
return r;
}
/* ********************************************************** */
// '[' expression ']' | simple quantifier?
public static boolean quantified(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "quantified")) return false;
boolean r;
Marker m = enter_section_(b, l, _COLLAPSE_, "<quantified>");
r = quantified_0(b, l + 1);
if (!r) r = quantified_1(b, l + 1);
exit_section_(b, l, m, BNF_QUANTIFIED, r, false, null);
return r;
}
// '[' expression ']'
private static boolean quantified_0(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "quantified_0")) return false;
boolean r;
Marker m = enter_section_(b);
r = consumeToken(b, BNF_LEFT_BRACKET);
r = r && expression(b, l + 1);
r = r && consumeToken(b, BNF_RIGHT_BRACKET);
exit_section_(b, m, null, r);
return r;
}
// simple quantifier?
private static boolean quantified_1(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "quantified_1")) return false;
boolean r;
Marker m = enter_section_(b);
r = simple(b, l + 1);
r = r && quantified_1_1(b, l + 1);
exit_section_(b, m, null, r);
return r;
}
// quantifier?
private static boolean quantified_1_1(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "quantified_1_1")) return false;
quantifier(b, l + 1);
return true;
}
/* ********************************************************** */
// '?' | '+' | '*'
public static boolean quantifier(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "quantifier")) return false;
boolean r;
Marker m = enter_section_(b, l, _NONE_, "<quantifier>");
r = consumeToken(b, BNF_OP_OPT);
if (!r) r = consumeToken(b, BNF_OP_ONEMORE);
if (!r) r = consumeToken(b, BNF_OP_ZEROMORE);
exit_section_(b, l, m, BNF_QUANTIFIER, r, false, null);
return r;
}
/* ********************************************************** */
// id
public static boolean reference_or_token(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "reference_or_token")) return false;
if (!nextTokenIs(b, BNF_ID)) return false;
boolean r;
Marker m = enter_section_(b);
r = consumeToken(b, BNF_ID);
exit_section_(b, m, BNF_REFERENCE_OR_TOKEN, r);
return r;
}
/* ********************************************************** */
// modifier* id '::=' expression attrs? ';'?
public static boolean rule(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "rule")) return false;
boolean r, p;
Marker m = enter_section_(b, l, _NONE_, "<rule>");
r = rule_0(b, l + 1);
r = r && consumeToken(b, BNF_ID);
r = r && consumeToken(b, BNF_OP_IS);
p = r; // pin = 3
r = r && report_error_(b, expression(b, l + 1));
r = p && report_error_(b, rule_4(b, l + 1)) && r;
r = p && rule_5(b, l + 1) && r;
exit_section_(b, l, m, BNF_RULE, r, p, rule_recover_until_parser_);
return r || p;
}
// modifier*
private static boolean rule_0(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "rule_0")) return false;
int c = current_position_(b);
while (true) {
if (!modifier(b, l + 1)) break;
if (!empty_element_parsed_guard_(b, "rule_0", c)) break;
c = current_position_(b);
}
return true;
}
// attrs?
private static boolean rule_4(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "rule_4")) return false;
attrs(b, l + 1);
return true;
}
// ';'?
private static boolean rule_5(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "rule_5")) return false;
consumeToken(b, BNF_SEMICOLON);
return true;
}
/* ********************************************************** */
// !'{'
static boolean rule_recover_until(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "rule_recover_until")) return false;
boolean r;
Marker m = enter_section_(b, l, _NOT_, null);
r = !consumeToken(b, BNF_LEFT_BRACE);
exit_section_(b, l, m, null, r, false, null);
return r;
}
/* ********************************************************** */
// option +
public static boolean sequence(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "sequence")) return false;
boolean r;
Marker m = enter_section_(b, l, _COLLAPSE_, "<sequence>");
r = option(b, l + 1);
int c = current_position_(b);
while (r) {
if (!option(b, l + 1)) break;
if (!empty_element_parsed_guard_(b, "sequence", c)) break;
c = current_position_(b);
}
exit_section_(b, l, m, BNF_SEQUENCE, r, false, null);
return r;
}
/* ********************************************************** */
// !(modifier* id '::=' ) reference_or_token | literal_expression | paren_expression
static boolean simple(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "simple")) return false;
boolean r;
Marker m = enter_section_(b);
r = simple_0(b, l + 1);
if (!r) r = literal_expression(b, l + 1);
if (!r) r = paren_expression(b, l + 1);
exit_section_(b, m, null, r);
return r;
}
// !(modifier* id '::=' ) reference_or_token
private static boolean simple_0(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "simple_0")) return false;
boolean r;
Marker m = enter_section_(b);
r = simple_0_0(b, l + 1);
r = r && reference_or_token(b, l + 1);
exit_section_(b, m, null, r);
return r;
}
// !(modifier* id '::=' )
private static boolean simple_0_0(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "simple_0_0")) return false;
boolean r;
Marker m = enter_section_(b, l, _NOT_, null);
r = !simple_0_0_0(b, l + 1);
exit_section_(b, l, m, null, r, false, null);
return r;
}
// modifier* id '::='
private static boolean simple_0_0_0(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "simple_0_0_0")) return false;
boolean r;
Marker m = enter_section_(b);
r = simple_0_0_0_0(b, l + 1);
r = r && consumeToken(b, BNF_ID);
r = r && consumeToken(b, BNF_OP_IS);
exit_section_(b, m, null, r);
return r;
}
// modifier*
private static boolean simple_0_0_0_0(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "simple_0_0_0_0")) return false;
int c = current_position_(b);
while (true) {
if (!modifier(b, l + 1)) break;
if (!empty_element_parsed_guard_(b, "simple_0_0_0_0", c)) break;
c = current_position_(b);
}
return true;
}
/* ********************************************************** */
// string
public static boolean string_literal_expression(PsiBuilder b, int l) {
if (!recursion_guard_(b, l, "string_literal_expression")) return false;
if (!nextTokenIs(b, BNF_STRING)) return false;
boolean r;
Marker m = enter_section_(b);
r = consumeToken(b, BNF_STRING);
exit_section_(b, m, BNF_STRING_LITERAL_EXPRESSION, r);
return r;
}
final static Parser attr_recover_until_parser_ = new Parser() {
public boolean parse(PsiBuilder b, int l) {
return attr_recover_until(b, l + 1);
}
};
final static Parser rule_recover_until_parser_ = new Parser() {
public boolean parse(PsiBuilder b, int l) {
return rule_recover_until(b, l + 1);
}
};
}

482
_testdata/Java/JFlexLexer.java Normal file
View File

@ -0,0 +1,482 @@
/* The following code was generated by JFlex 1.4.3 on 28/01/16 11:27 */
package test;
import com.intellij.lexer.*;
import com.intellij.psi.tree.IElementType;
import static org.intellij.grammar.psi.BnfTypes.*;
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.3
* on 28/01/16 11:27 from the specification file
* <tt>/home/abigail/code/intellij-grammar-kit-test/src/test/_GrammarLexer.flex</tt>
*/
public class _GrammarLexer implements FlexLexer {
/** initial size of the lookahead buffer */
private static final int ZZ_BUFFERSIZE = 16384;
/** lexical states */
public static final int YYINITIAL = 0;
/**
* ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l
* ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l
* at the beginning of a line
* l is of the form l = 2*k, k a non negative integer
*/
private static final int ZZ_LEXSTATE[] = {
0, 0
};
/**
* Translates characters to character classes
*/
private static final String ZZ_CMAP_PACKED =
"\11\0\1\1\1\1\1\0\1\1\1\1\22\0\1\1\101\0\1\13"+
"\1\0\1\3\1\14\1\0\1\10\1\0\1\2\3\0\1\12\1\7"+
"\3\0\1\6\1\4\1\5\1\11\uff8a\0";
/**
* Translates characters to character classes
*/
private static final char [] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED);
/**
* Translates DFA states to action switch labels.
*/
private static final int [] ZZ_ACTION = zzUnpackAction();
private static final String ZZ_ACTION_PACKED_0 =
"\1\0\1\1\1\2\3\1\1\3\10\0\1\4\1\5";
private static int [] zzUnpackAction() {
int [] result = new int[17];
int offset = 0;
offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
return result;
}
private static int zzUnpackAction(String packed, int offset, int [] result) {
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int count = packed.charAt(i++);
int value = packed.charAt(i++);
do result[j++] = value; while (--count > 0);
}
return j;
}
/**
* Translates a state to a row index in the transition table
*/
private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
private static final String ZZ_ROWMAP_PACKED_0 =
"\0\0\0\15\0\32\0\47\0\64\0\101\0\15\0\116"+
"\0\133\0\150\0\165\0\202\0\217\0\234\0\251\0\15"+
"\0\15";
private static int [] zzUnpackRowMap() {
int [] result = new int[17];
int offset = 0;
offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
return result;
}
private static int zzUnpackRowMap(String packed, int offset, int [] result) {
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int high = packed.charAt(i++) << 16;
result[j++] = high | packed.charAt(i++);
}
return j;
}
/**
* The transition table of the DFA
*/
private static final int [] ZZ_TRANS = zzUnpackTrans();
private static final String ZZ_TRANS_PACKED_0 =
"\1\2\1\3\1\4\1\2\1\5\2\2\1\6\5\2"+
"\16\0\1\3\16\0\1\7\16\0\1\10\20\0\1\11"+
"\11\0\1\12\20\0\1\13\4\0\1\14\25\0\1\15"+
"\10\0\1\16\21\0\1\17\10\0\1\20\12\0\1\21"+
"\6\0";
private static int [] zzUnpackTrans() {
int [] result = new int[182];
int offset = 0;
offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
return result;
}
private static int zzUnpackTrans(String packed, int offset, int [] result) {
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int count = packed.charAt(i++);
int value = packed.charAt(i++);
value--;
do result[j++] = value; while (--count > 0);
}
return j;
}
/* error codes */
private static final int ZZ_UNKNOWN_ERROR = 0;
private static final int ZZ_NO_MATCH = 1;
private static final int ZZ_PUSHBACK_2BIG = 2;
private static final char[] EMPTY_BUFFER = new char[0];
private static final int YYEOF = -1;
private static java.io.Reader zzReader = null; // Fake
/* error messages for the codes above */
private static final String ZZ_ERROR_MSG[] = {
"Unkown internal scanner error",
"Error: could not match input",
"Error: pushback value was too large"
};
/**
* ZZ_ATTRIBUTE[aState] contains the attributes of state <code>aState</code>
*/
private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
private static final String ZZ_ATTRIBUTE_PACKED_0 =
"\1\0\1\11\4\1\1\11\10\0\2\11";
private static int [] zzUnpackAttribute() {
int [] result = new int[17];
int offset = 0;
offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
return result;
}
private static int zzUnpackAttribute(String packed, int offset, int [] result) {
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int count = packed.charAt(i++);
int value = packed.charAt(i++);
do result[j++] = value; while (--count > 0);
}
return j;
}
/** the current state of the DFA */
private int zzState;
/** the current lexical state */
private int zzLexicalState = YYINITIAL;
/** this buffer contains the current text to be matched and is
the source of the yytext() string */
private CharSequence zzBuffer = "";
/** this buffer may contains the current text array to be matched when it is cheap to acquire it */
private char[] zzBufferArray;
/** the textposition at the last accepting state */
private int zzMarkedPos;
/** the textposition at the last state to be included in yytext */
private int zzPushbackPos;
/** the current text position in the buffer */
private int zzCurrentPos;
/** startRead marks the beginning of the yytext() string in the buffer */
private int zzStartRead;
/** endRead marks the last character in the buffer, that has been read
from input */
private int zzEndRead;
/**
* zzAtBOL == true <=> the scanner is currently at the beginning of a line
*/
private boolean zzAtBOL = true;
/** zzAtEOF == true <=> the scanner is at the EOF */
private boolean zzAtEOF;
/* user code: */
public _GrammarLexer() {
this((java.io.Reader)null);
}
/**
* Creates a new scanner
*
* @param in the java.io.Reader to read input from.
*/
public _GrammarLexer(java.io.Reader in) {
this.zzReader = in;
}
/**
* Unpacks the compressed character translation table.
*
* @param packed the packed character translation table
* @return the unpacked character translation table
*/
private static char [] zzUnpackCMap(String packed) {
char [] map = new char[0x10000];
int i = 0; /* index in packed string */
int j = 0; /* index in unpacked array */
while (i < 52) {
int count = packed.charAt(i++);
char value = packed.charAt(i++);
do map[j++] = value; while (--count > 0);
}
return map;
}
public final int getTokenStart(){
return zzStartRead;
}
public final int getTokenEnd(){
return getTokenStart() + yylength();
}
public void reset(CharSequence buffer, int start, int end,int initialState){
zzBuffer = buffer;
zzBufferArray = com.intellij.util.text.CharArrayUtil.fromSequenceWithoutCopying(buffer);
zzCurrentPos = zzMarkedPos = zzStartRead = start;
zzPushbackPos = 0;
zzAtEOF = false;
zzAtBOL = true;
zzEndRead = end;
yybegin(initialState);
}
/**
* Refills the input buffer.
*
* @return <code>false</code>, iff there was new input.
*
* @exception java.io.IOException if any I/O-Error occurs
*/
private boolean zzRefill() throws java.io.IOException {
return true;
}
/**
* Returns the current lexical state.
*/
public final int yystate() {
return zzLexicalState;
}
/**
* Enters a new lexical state
*
* @param newState the new lexical state
*/
public final void yybegin(int newState) {
zzLexicalState = newState;
}
/**
* Returns the text matched by the current regular expression.
*/
public final CharSequence yytext() {
return zzBuffer.subSequence(zzStartRead, zzMarkedPos);
}
/**
* Returns the character at position <tt>pos</tt> from the
* matched text.
*
* It is equivalent to yytext().charAt(pos), but faster
*
* @param pos the position of the character to fetch.
* A value from 0 to yylength()-1.
*
* @return the character at position pos
*/
public final char yycharat(int pos) {
return zzBufferArray != null ? zzBufferArray[zzStartRead+pos]:zzBuffer.charAt(zzStartRead+pos);
}
/**
* Returns the length of the matched text region.
*/
public final int yylength() {
return zzMarkedPos-zzStartRead;
}
/**
* Reports an error that occured while scanning.
*
* In a wellformed scanner (no or only correct usage of
* yypushback(int) and a match-all fallback rule) this method
* will only be called with things that "Can't Possibly Happen".
* If this method is called, something is seriously wrong
* (e.g. a JFlex bug producing a faulty scanner etc.).
*
* Usual syntax/scanner level error handling should be done
* in error fallback rules.
*
* @param errorCode the code of the errormessage to display
*/
private void zzScanError(int errorCode) {
String message;
try {
message = ZZ_ERROR_MSG[errorCode];
}
catch (ArrayIndexOutOfBoundsException e) {
message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
}
throw new Error(message);
}
/**
* Pushes the specified amount of characters back into the input stream.
*
* They will be read again by then next call of the scanning method
*
* @param number the number of characters to be read again.
* This number must not be greater than yylength()!
*/
public void yypushback(int number) {
if ( number > yylength() )
zzScanError(ZZ_PUSHBACK_2BIG);
zzMarkedPos -= number;
}
/**
* Resumes scanning until the next regular expression is matched,
* the end of input is encountered or an I/O-Error occurs.
*
* @return the next token
* @exception java.io.IOException if any I/O-Error occurs
*/
public IElementType advance() throws java.io.IOException {
int zzInput;
int zzAction;
// cached fields:
int zzCurrentPosL;
int zzMarkedPosL;
int zzEndReadL = zzEndRead;
CharSequence zzBufferL = zzBuffer;
char[] zzBufferArrayL = zzBufferArray;
char [] zzCMapL = ZZ_CMAP;
int [] zzTransL = ZZ_TRANS;
int [] zzRowMapL = ZZ_ROWMAP;
int [] zzAttrL = ZZ_ATTRIBUTE;
while (true) {
zzMarkedPosL = zzMarkedPos;
zzAction = -1;
zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
zzState = ZZ_LEXSTATE[zzLexicalState];
zzForAction: {
while (true) {
if (zzCurrentPosL < zzEndReadL)
zzInput = (zzBufferArrayL != null ? zzBufferArrayL[zzCurrentPosL++] : zzBufferL.charAt(zzCurrentPosL++));
else if (zzAtEOF) {
zzInput = YYEOF;
break zzForAction;
}
else {
// store back cached positions
zzCurrentPos = zzCurrentPosL;
zzMarkedPos = zzMarkedPosL;
boolean eof = zzRefill();
// get translated positions and possibly new buffer
zzCurrentPosL = zzCurrentPos;
zzMarkedPosL = zzMarkedPos;
zzBufferL = zzBuffer;
zzEndReadL = zzEndRead;
if (eof) {
zzInput = YYEOF;
break zzForAction;
}
else {
zzInput = (zzBufferArrayL != null ? zzBufferArrayL[zzCurrentPosL++] : zzBufferL.charAt(zzCurrentPosL++));
}
}
int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ];
if (zzNext == -1) break zzForAction;
zzState = zzNext;
int zzAttributes = zzAttrL[zzState];
if ( (zzAttributes & 1) == 1 ) {
zzAction = zzState;
zzMarkedPosL = zzCurrentPosL;
if ( (zzAttributes & 8) == 8 ) break zzForAction;
}
}
}
// store back cached position
zzMarkedPos = zzMarkedPosL;
switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
case 1:
{ return com.intellij.psi.TokenType.BAD_CHARACTER;
}
case 6: break;
case 4:
{ return BNF_STRING;
}
case 7: break;
case 5:
{ return BNF_NUMBER;
}
case 8: break;
case 3:
{ return BNF_ID;
}
case 9: break;
case 2:
{ return com.intellij.psi.TokenType.WHITE_SPACE;
}
case 10: break;
default:
if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
zzAtEOF = true;
return null;
}
else {
zzScanError(ZZ_NO_MATCH);
}
}
}
}
}

18
_testdata/JavaScript/jquery-1.6.1.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

93
_testdata/JavaScript/namespace.js Normal file
View File

@ -0,0 +1,93 @@
(function(root, factory) {
if (typeof define === 'function' && define.amd) {
define(['lodash'], factory);
} else if (typeof exports !== 'undefined') {
module.exports = factory(require('lodash'));
} else {
root.Namespace = factory(root._);
}
})(this, function(_) {
'use strict';
/**
* @module namespace
* @class namespace
*/
function Namespace() {}
/**
* Regex for splitting keypaths into arrays.
*
* @private
* @const {RegExp}
* @type
*/
var KEYPATH_SPLITTER = /\./g;
/**
* An internal cache to avoid calculating a keypath more than once.
*
* @private
* @type {Object}
*/
var _keypaths = {};
_.extend(Namespace.prototype, {
/**
* Adds a definition to the namespace object.
*
* @public
* @instance
* @method add
* @param {String} keypath - The keypath for the definition to be added at.
* @param {Function|Object} definition - The definition to be added.
* @return {Function|Object} - The definition.
*/
add: function(keypath, definition) {
return this._walk(keypath, function(memo, name, index, keypath) {
if (index + 1 === keypath.length) {
memo[name] = _.extend(definition, memo[name]);
}
return memo[name] || (memo[name] = {});
});
},
/**
* Retrieves a definition from the namespace safely.
*
* @public
* @instance
* @method get
* @param {String} keypath - The keypath to lookup a definition for.
* @returns {Function|Object|undefined} - The definition if it exists, otherwise `undefined`.
*/
get: function(keypath) {
return this._walk(keypath);
},
/**
* An internal function for walking a keypath.
*
* @private
* @instance
* @method _walk
* @param {String} keypath - The keypath to walk through.
* @param {Function} [callback] - An optional callback to be called at each item in the path.
* @returns {function|Object|undefined} - The reduced keypath.
*/
_walk: function(keypath, callback) {
return _.reduce(
_keypaths[keypath] || (_keypaths[keypath] = keypath.split(KEYPATH_SPLITTER)),
callback || function(memo, name) {
return memo && memo[name];
},
this
);
}
});
return Namespace;
});
//# sourceMappingURL=namespace.js.map

78
_testdata/PostScript/lambda.pfa Normal file
View File

@ -0,0 +1,78 @@
%!PS-AdobeFont-1.0: Greek_Lambda_Character-Regular 020.017
%%Title: Greek_Lambda_Character-Regular
%Version: 020.017
%%CreationDate: Sun Jul 23 23:14:02 2017
%%Creator: John Gardner /ThatGuyLinguistWatchersAreSickOf
%Copyright: NONE. NADA. PUBLIC DOMAIN, BOI
% Generated by FontForge 20170719 (http://fontforge.sf.net/)
%%EndComments
10 dict begin
/FontType 1 def
/FontMatrix [0.000488281 0 0 0.000488281 0 0 ]readonly def
/FontName /Greek_Lambda_Character-Regular def
/FontBBox {68 -362 1158 1556 }readonly def
/PaintType 0 def
/FontInfo 11 dict dup begin
/version (020.017) readonly def
/Notice (NONE. NADA. PUBLIC DOMAIN, BOI) readonly def
/FullName (Greek_Lambda_Character Regular) readonly def
/FamilyName (Greek_Lambda_Character) readonly def
/Weight (Regular) readonly def
/FSType 8 def
/ItalicAngle 0 def
/isFixedPitch false def
/UnderlinePosition -175 def
/UnderlineThickness 90 def
/ascent 1556 def
end readonly def
/Encoding 256 array
0 1 255 { 1 index exch /.notdef put} for
dup 13/uni000D put
dup 32/space put
readonly def
currentdict end
currentfile eexec
743F8413F3636CA85A9FFEFB50B4BB27302A5F6C876586CCC1670A7EF5521E6ADE15AAB4
DD2DDDB83735311FC63DB80D2C96AECFA05BB67F865EA35934B4B79A203A8DD489B09C79
FF6EB9DBCFD889C3E73F8C94BC342AF671D6F688870A62EE1A0DF216E150FFEC64A8C2B7
509AD05C011599C1AD84E6C4B668E07EA219BD72663D8AF4CA8EC8E23AA90DE90BE940C6
6DB849CEDB3B64961365A7CCE47F4FC9E30FDEE4B14B90C2E0D8C344EBC974EABF417B3D
28251A78ACEE2BFC4212B1E3E9C7EBC3262821EE98E538713C64DF0BC13C19337B1307DE
427F2C8C4FF2126B9FF99DDDFB332AF3EBCD23349171C5C7E66B1C53EF39C35516C0DC7D
D6F70D59C28603519559B710C0128C22BA268D330AE02690534939E5B58BAAB00A59B0E6
E515E80A04F2B315A00FB1CA49A4B0259167BBA9852544092955FE0883DBCD89B739AA94
8C6277B54019E9870B44E5A998F0F2175545F602CC4630E4BE01E35F4CEFDD7D1B737358
1B2D6F612B23AFAF221AB18CD868E7548EE5064762971F8868940F26134CA7BB40734220
06EEBE10B5F5251FE36B0E794D8B0D7AC814F4C30BB5A0435C203748B40217BEAF506328
D30E9417382B228FF6DF97A71135B613CC3EF4CDD8276711842C3DB0D31A25883544FFF5
D8719874AD10816F3EB272D5C245440127B9F2967DC46EAF4BED77FF2C2585A355EB586E
09BE8BA3D104EA174EA004DCE00066F7606EEF55376E332566235B8F4283D63E2AE5F2E3
6A6940BED4FA83C6E82C444B2D790BAA36E641908C61F6409CA09D9723B2F169BE42B166
5E704AA69D594B2A22C631D030029B772C3C200A0A483A13403E2EF19274B73860FFEA37
521C65865A79A5429EED01C0F95AB6F66F030BF14CCB42A6DA2435B74135BB5D35EF6DC6
FE48D29690417795B9D7AA2A3D66B7A175F8B6C23AFB5A472343C2DFDCC242C27033DD6F
B65B1CD5DC29B9DA455FAAC3A9408BB9B61C2B701741C34CE88EBFF05B7045CBEAC8788E
548F0E642277D4F9AED30A3431070805910970243FD069130D641AFAE28754B1355618B8
706D163027980F2A76A59E0B9D2FB92BF17943129A8A6FB15CBF6D376C464329E24A118B
8E38987CDD4CD5F214B26D65BEEDB2ECD7502861747089298A653654062A3AD5AC4FC4AF
5DEF7BDD4C5CDBC828A1F12235F68522B45BE0C2D6FB663B51E76BF559815F41C25EC7F8
17A92B9E139F3E6D27A8512A261EE576459DE603F386516E46B8326574E900101A8D49C9
E8798E734F56C4E2D78B14D6B14E96F3523F3F34464235C96F94609852F90A0A1B51CE14
25EC76E65E5D1C5D2EEC730FC438F4083A806C67D0A1566D0E6EF70DC38E64DCA44DFA66
20073E848BDFDB2B793AFE29B47917D87CB8320F48A7C083CAAD2CA20C25D705D78B33D5
A5EAB8C671A57BBC3C896DE54C71B8F860B640B7C30199A2443353E3727FA99C168DB4E8
ABE021B7DDB526E30B2075028BD8D387D32E025E458D05E3A25B9BC94ED54F7A0FCA3D61
604D2F49A205CBDF5FBDD5D3E1D31C6F7310CC155982689FA1F5723DDAE2EDEB5E69F03C
4CCDBBCB348E57AF222744A07DCCE69236EB29499A87C0A201EDD6A402EA23C035732184
339049E8C101388304CD20EAA8554A9B6E4F2B126A0593F4401992E73F7BFEF89DE3DFDE
DA548E3EE05D5B4EE0D114F916
0000000000000000000000000000000000000000000000000000000000000000
0000000000000000000000000000000000000000000000000000000000000000
0000000000000000000000000000000000000000000000000000000000000000
0000000000000000000000000000000000000000000000000000000000000000
0000000000000000000000000000000000000000000000000000000000000000
0000000000000000000000000000000000000000000000000000000000000000
0000000000000000000000000000000000000000000000000000000000000000
0000000000000000000000000000000000000000000000000000000000000000
cleartomark

72
_testdata/R/import.Rd Normal file
View File

@ -0,0 +1,72 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/hello.R
\name{import}
\alias{import}
\title{Import a module into the current scope}
\usage{
import(module, attach, attach_operators = TRUE)
}
\arguments{
\item{module}{an identifier specifying the full module path}
\item{attach}{if \code{TRUE}, attach the newly loaded module to the object
search path (see \code{Details})}
\item{attach_operators}{if \code{TRUE}, attach operators of module to the
object search path, even if \code{attach} is \code{FALSE}}
}
\value{
the loaded module environment (invisible)
}
\description{
\code{module = import('module')} imports a specified module and makes its
code available via the environment-like object it returns.
}
\details{
Modules are loaded in an isolated environment which is returned, and
optionally attached to the object search path of the current scope (if
argument \code{attach} is \code{TRUE}).
\code{attach} defaults to \code{FALSE}. However, in interactive code it is
often helpful to attach packages by default. Therefore, in interactive code
invoked directly from the terminal only (i.e. not within modules),
\code{attach} defaults to the value of \code{options('import.attach')}, which
can be set to \code{TRUE} or \code{FALSE} depending on the user’s preference.
\code{attach_operators} causes \emph{operators} to be attached by default,
because operators can only be invoked in R if they re found in the search
path. Not attaching them therefore drastically limits a module’s usefulness.
Modules are searched in the module search path \code{options('import.path')}.
This is a vector of paths to consider, from the highest to the lowest
priority. The current directory is \emph{always} considered first. That is,
if a file \code{a.r} exists both in the current directory and in a module
search path, the local file \code{./a.r} will be loaded.
Module names can be fully qualified to refer to nested paths. See
\code{Examples}.
}
\note{
Unlike for packages, attaching happens \emph{locally}: if
\code{import} is executed in the global environment, the effect is the same.
Otherwise, the imported module is inserted as the parent of the current
\code{environment()}. When used (globally) \emph{inside} a module, the newly
imported module is only available inside the module’s search path, not
outside it (nor in other modules which might be loaded).
}
\examples{
# `a.r` is a file in the local directory containing a function `f`.
a = import('a')
a$f()
# b/c.r is a file in path `b`, containing a function `g`.
import('b/c', attach = TRUE)
g() # No module name qualification necessary
}
\seealso{
\code{unload}
\code{reload}
\code{module_name}
}

196
benchmark_test.go Normal file
View File

@ -0,0 +1,196 @@
package enry
import (
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"testing"
)
type sample struct {
filename string
content []byte
}
var (
slow bool
overcomeLanguage string
overcomeLanguages []string
samples []*sample
)
func TestMain(m *testing.M) {
flag.BoolVar(&slow, "slow", false, "run benchmarks per sample for strategies too")
flag.Parse()
tmpLinguistDir, cleanupNeeded, err := maybeCloneLinguist()
if err != nil {
log.Fatal(err)
}
if cleanupNeeded {
defer os.RemoveAll(tmpLinguistDir)
}
samplesDir := filepath.Join(tmpLinguistDir, "samples")
samples, err = getSamples(samplesDir)
if err != nil {
log.Fatal(err)
}
os.Exit(m.Run())
}
func getSamples(dir string) ([]*sample, error) {
samples := make([]*sample, 0, 2000)
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
content, err := ioutil.ReadFile(path)
if err != nil {
return err
}
s := &sample{
filename: path,
content: content,
}
samples = append(samples, s)
return nil
})
return samples, err
}
func BenchmarkGetLanguageTotal(b *testing.B) {
if slow {
b.SkipNow()
}
var o string
b.Run("GetLanguage()_TOTAL", func(b *testing.B) {
for n := 0; n < b.N; n++ {
for _, sample := range samples {
o = GetLanguage(sample.filename, sample.content)
}
}
overcomeLanguage = o
})
}
func BenchmarkClassifyTotal(b *testing.B) {
if slow {
b.SkipNow()
}
var o []string
b.Run("Classify()_TOTAL", func(b *testing.B) {
for n := 0; n < b.N; n++ {
for _, sample := range samples {
o = defaultClassifier.classify(sample.content, nil)
}
overcomeLanguages = o
}
})
}
func BenchmarkStrategiesTotal(b *testing.B) {
if slow {
b.SkipNow()
}
benchmarks := benchmarkForAllStrategies("TOTAL")
var o []string
for _, benchmark := range benchmarks {
b.Run(benchmark.name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
for _, sample := range samples {
o = benchmark.strategy(sample.filename, sample.content, benchmark.candidates)
}
overcomeLanguages = o
}
})
}
}
func BenchmarkGetLanguagePerSample(b *testing.B) {
if !slow {
b.SkipNow()
}
var o string
for _, sample := range samples {
b.Run("GetLanguage()_SAMPLE_"+sample.filename, func(b *testing.B) {
for n := 0; n < b.N; n++ {
o = GetLanguage(sample.filename, sample.content)
}
overcomeLanguage = o
})
}
}
func BenchmarkClassifyPerSample(b *testing.B) {
if !slow {
b.SkipNow()
}
var o []string
for _, sample := range samples {
b.Run("Classify()_SAMPLE_"+sample.filename, func(b *testing.B) {
for n := 0; n < b.N; n++ {
o = defaultClassifier.classify(sample.content, nil)
}
overcomeLanguages = o
})
}
}
func BenchmarkStrategiesPerSample(b *testing.B) {
if !slow {
b.SkipNow()
}
benchmarks := benchmarkForAllStrategies("SAMPLE")
var o []string
for _, benchmark := range benchmarks {
for _, sample := range samples {
b.Run(benchmark.name+sample.filename, func(b *testing.B) {
for n := 0; n < b.N; n++ {
o = benchmark.strategy(sample.filename, sample.content, benchmark.candidates)
}
overcomeLanguages = o
})
}
}
}
type strategyName struct {
name string
strategy Strategy
candidates []string
}
func benchmarkForAllStrategies(class string) []strategyName {
return []strategyName{
{name: fmt.Sprintf("GetLanguagesByModeline()_%s_", class), strategy: GetLanguagesByModeline},
{name: fmt.Sprintf("GetLanguagesByFilename()_%s_", class), strategy: GetLanguagesByFilename},
{name: fmt.Sprintf("GetLanguagesByShebang()_%s_", class), strategy: GetLanguagesByShebang},
{name: fmt.Sprintf("GetLanguagesByExtension()_%s_", class), strategy: GetLanguagesByExtension},
{name: fmt.Sprintf("GetLanguagesByContent()_%s_", class), strategy: GetLanguagesByContent},
}
}

6
benchmarks/csv/enry-distribution.csv Normal file
View File

@ -0,0 +1,6 @@
timeInterval,enry,numberOfFiles
1us-10us,enry,83
10us-100us,enry,1341
100us-1ms,enry,314
1ms-10ms,enry,146
10ms-100ms,enry,48
1 timeInterval enry numberOfFiles
2 1us-10us enry 83
3 10us-100us enry 1341
4 100us-1ms enry 314
5 1ms-10ms enry 146
6 10ms-100ms enry 48

13532
benchmarks/csv/enry-samples.csv Normal file
View File

File diff suppressed because it is too large Load Diff

8
benchmarks/csv/enry-total.csv Normal file
View File

@ -0,0 +1,8 @@
function,tool,iterations,ns/op
GetLanguage(),enry,100,2333748307
Classify(),enry,3,53842505853
GetLanguagesByModeline(),enry,1000,228234491
GetLanguagesByFilename(),enry,1000000,124782
GetLanguagesByShebang(),enry,100000,2339138
GetLanguagesByExtension(),enry,200000,1110007
GetLanguagesByContent(),enry,500,342358978
1 function tool iterations ns/op
2 GetLanguage() enry 100 2333748307
3 Classify() enry 3 53842505853
4 GetLanguagesByModeline() enry 1000 228234491
5 GetLanguagesByFilename() enry 1000000 124782
6 GetLanguagesByShebang() enry 100000 2339138
7 GetLanguagesByExtension() enry 200000 1110007
8 GetLanguagesByContent() enry 500 342358978

6
benchmarks/csv/linguist-distribution.csv Normal file
View File

@ -0,0 +1,6 @@
timeInterval,linguist,numberOfFiles
1us-10us,linguist,0
10us-100us,linguist,120
100us-1ms,linguist,1070
1ms-10ms,linguist,816
10ms-100ms,linguist,71
1 timeInterval linguist numberOfFiles
2 1us-10us linguist 0
3 10us-100us linguist 120
4 100us-1ms linguist 1070
5 1ms-10ms linguist 816
6 10ms-100ms linguist 71

14554
benchmarks/csv/linguist-samples.csv Normal file
View File

File diff suppressed because it is too large Load Diff

8
benchmarks/csv/linguist-total.csv Normal file
View File

@ -0,0 +1,8 @@
function,tool,iterations,ns/op
GetLanguage(),linguist,5,3822076000
Classify(),linguist,5,329660597600
GetLanguagesByModeline(),linguist,5,2770912600
GetLanguagesByFilename(),linguist,5,34159000
GetLanguagesByShebang(),linguist,5,159317200
GetLanguagesByExtension(),linguist,5,354929800
GetLanguagesByContent(),linguist,5,3881611000
1 function tool iterations ns/op
2 GetLanguage() linguist 5 3822076000
3 Classify() linguist 5 329660597600
4 GetLanguagesByModeline() linguist 5 2770912600
5 GetLanguagesByFilename() linguist 5 34159000
6 GetLanguagesByShebang() linguist 5 159317200
7 GetLanguagesByExtension() linguist 5 354929800
8 GetLanguagesByContent() linguist 5 3881611000

BIN
benchmarks/histogram/distribution.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 16 KiB

126
benchmarks/linguist-samples.rb Executable file
View File

@ -0,0 +1,126 @@
#!/usr/bin/env ruby
require 'benchmark'
require 'linguist'
iterations = (ARGV[0] || 1).to_i
# BenchBlob wraps a FileBlob to keep data loaded and to clean attributes added by language detection.
class BenchBlob < Linguist::FileBlob
attr_accessor :data
def initialize(path, base_path = nil)
super
@data = File.read(@fullpath)
end
def clean
@_mime_type = nil
@detect_encoding = nil
@lines = nil
end
end
def get_samples(root)
samples = Array.new
Dir.foreach(root) do |file|
path = File.join(root, file)
if file == "." or file == ".."
next
elsif File.directory?(path)
get_samples(path).each do |blob|
samples << blob
end
else
samples << BenchBlob.new(path)
end
end
return samples
end
samples = get_samples('.linguist/samples')
languages = Linguist::Language.all
samples.each do |blob|
sample_name = blob.path.gsub(/\s/, '_')
Benchmark.bmbm do |bm|
bm.report('GetLanguage()_SAMPLE_' + sample_name + ' ' + iterations.to_s) do
iterations.times do
Linguist::detect(blob)
blob.clean
end
end
end
end
samples.each do |blob|
sample_name = blob.path.gsub(/\s/, '_')
Benchmark.bmbm do |bm|
bm.report('Classify()_SAMPLE_' + sample_name + ' ' + iterations.to_s) do
iterations.times do
Linguist::Classifier.classify(Linguist::Samples.cache, blob.data)
blob.clean
end
end
end
end
samples.each do |blob|
sample_name = blob.path.gsub(/\s/, '_')
Benchmark.bmbm do |bm|
bm.report('GetLanguagesByModeline()_SAMPLE_' + sample_name + ' ' + iterations.to_s) do
iterations.times do
Linguist::Strategy::Modeline.call(blob, languages)
blob.clean
end
end
end
end
samples.each do |blob|
sample_name = blob.path.gsub(/\s/, '_')
Benchmark.bmbm do |bm|
bm.report('GetLanguagesByFilename()_SAMPLE_' + sample_name + ' ' + iterations.to_s) do
iterations.times do
Linguist::Strategy::Filename.call(blob, languages)
blob.clean
end
end
end
end
samples.each do |blob|
sample_name = blob.path.gsub(/\s/, '_')
Benchmark.bmbm do |bm|
bm.report('GetLanguagesByShebang()_SAMPLE_' + sample_name + ' ' + iterations.to_s) do
iterations.times do
Linguist::Shebang.call(blob, languages)
blob.clean
end
end
end
end
samples.each do |blob|
sample_name = blob.path.gsub(/\s/, '_')
Benchmark.bmbm do |bm|
bm.report('GetLanguagesByExtension()_SAMPLE_' + sample_name + ' ' + iterations.to_s) do
iterations.times do
Linguist::Strategy::Extension.call(blob, languages)
blob.clean
end
end
end
end
samples.each do |blob|
sample_name = blob.path.gsub(/\s/, '_')
Benchmark.bmbm do |bm|
bm.report('GetLanguagesByContent()_SAMPLE_' + sample_name + ' ' + iterations.to_s) do
iterations.times do
Linguist::Heuristics.call(blob, languages)
blob.clean
end
end
end
end

120
benchmarks/linguist-total.rb Executable file
View File

@ -0,0 +1,120 @@
#!/usr/bin/env ruby
require 'benchmark'
require 'linguist'
iterations = (ARGV[0] || 1).to_i
# BenchBlob wraps a FileBlob to keep data loaded and to clean attributes added by language detection.
class BenchBlob < Linguist::FileBlob
attr_accessor :data
attr_accessor :fullpath
def initialize(path, base_path = nil)
super
@data = File.read(@fullpath)
end
def clean
@_mime_type = nil
@detect_encoding = nil
@lines = nil
end
end
def get_samples(root)
samples = Array.new
Dir.foreach(root) do |file|
path = File.join(root, file)
if file == "." or file == ".."
next
elsif File.directory?(path)
get_samples(path).each do |blob|
samples << blob
end
else
samples << BenchBlob.new(path)
end
end
return samples
end
samples = get_samples('.linguist/samples')
languages = Linguist::Language.all
Benchmark.bmbm do |bm|
time = bm.report('GetLanguage()_TOTAL ' + iterations.to_s) do
iterations.times do
samples.each do |blob|
Linguist::detect(blob)
blob.clean
end
end
end
end
Benchmark.bmbm do |bm|
bm.report('Classify()_TOTAL ' + iterations.to_s) do
iterations.times do
samples.each do |blob|
Linguist::Classifier.classify(Linguist::Samples.cache, blob.data)
blob.clean
end
end
end
end
Benchmark.bmbm do |bm|
bm.report('GetLanguagesByModeline()_TOTAL ' + iterations.to_s) do
iterations.times do
samples.each do |blob|
Linguist::Strategy::Modeline.call(blob, languages)
blob.clean
end
end
end
end
Benchmark.bmbm do |bm|
bm.report('GetLanguagesByFilename()_TOTAL ' + iterations.to_s) do
iterations.times do
samples.each do |blob|
Linguist::Strategy::Filename.call(blob, languages)
blob.clean
end
end
end
end
Benchmark.bmbm do |bm|
bm.report('GetLanguagesByShebang()_TOTAL ' + iterations.to_s) do
iterations.times do
samples.each do |blob|
Linguist::Shebang.call(blob, languages)
blob.clean
end
end
end
end
Benchmark.bmbm do |bm|
bm.report('GetLanguagesByExtension()_TOTAL ' + iterations.to_s) do
iterations.times do
samples.each do |blob|
Linguist::Strategy::Extension.call(blob, languages)
blob.clean
end
end
end
end
Benchmark.bmbm do |bm|
bm.report('GetLanguagesByContent()_TOTAL ' + iterations.to_s) do
iterations.times do
samples.each do |blob|
Linguist::Heuristics.call(blob, languages)
blob.clean
end
end
end
end

7
benchmarks/parse.sh Executable file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -e
cd benchmarks/output
go run ../parser/main.go -outdir ../csv
cd ../csv
go run ../parser/main.go -distribution

386
benchmarks/parser/main.go Normal file
View File

@ -0,0 +1,386 @@
package main
import (
"bufio"
"bytes"
"encoding/csv"
"flag"
"fmt"
"io/ioutil"
"log"
"math"
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
)
const (
// functions benchmarked
getLanguageFunc = "GetLanguage()"
classifyFunc = "Classify()"
modelineFunc = "GetLanguagesByModeline()"
filenameFunc = "GetLanguagesByFilename()"
shebangFunc = "GetLanguagesByShebang()"
extensionFunc = "GetLanguagesByExtension()"
contentFunc = "GetLanguagesByContent()"
// benchmark's outputs
enryTotalBench = "enry_total.bench"
enrySamplesBench = "enry_samples.bench"
linguistTotalBench = "linguist_total.bench"
linguistSamplesBench = "linguist_samples.bench"
// files to generate
enryTotalCSV = "enry-total.csv"
enrySamplesCSV = "enry-samples.csv"
linguistTotalCSV = "linguist-total.csv"
linguistSamplesCSV = "linguist-samples.csv"
// files to generate with flag distribution
enryDistributionCSV = "enry-distribution.csv"
linguistDistributionCSV = "linguist-distribution.csv"
)
var (
// flags
distribution bool
outDir string
enryFunctions = []string{getLanguageFunc, classifyFunc, modelineFunc, filenameFunc, shebangFunc, extensionFunc, contentFunc}
distributionIntervals = []string{"1us-10us", "10us-100us", "100us-1ms", "1ms-10ms", "10ms-100ms"}
)
func main() {
flag.BoolVar(&distribution, "distribution", false, "generate enry-distribution.csv and linguist-distribution.csv")
flag.StringVar(&outDir, "outdir", "", "path to leave csv files")
flag.Parse()
if distribution {
generateDistributionCSV()
return
}
generateCSV()
}
func generateDistributionCSV() {
CSVFiles := []struct {
in string
out string
tool string
}{
{in: enrySamplesCSV, out: enryDistributionCSV, tool: "enry"},
{in: linguistSamplesCSV, out: linguistDistributionCSV, tool: "linguist"},
}
for _, CSVFile := range CSVFiles {
f, err := os.Open(CSVFile.in)
if err != nil {
log.Println(err)
continue
}
defer f.Close()
r := csv.NewReader(f)
CSVSamples, err := r.ReadAll()
if err != nil {
log.Println(err)
continue
}
CSVDistribution, err := buildDistribution(CSVSamples[1:], CSVFile.tool)
if err != nil {
log.Println(err)
continue
}
if err := writeCSV(CSVDistribution, filepath.Join(outDir, CSVFile.out)); err != nil {
log.Println(err)
continue
}
}
}
func buildDistribution(CSVSamples [][]string, tool string) ([][]string, error) {
count := make(map[string]int, len(distributionIntervals))
for _, row := range CSVSamples {
if row[1] != getLanguageFunc {
continue
}
num, err := strconv.ParseFloat(row[len(row)-1], 64)
if err != nil {
return nil, err
}
arrangeByTime(count, num)
}
CSVDistribution := make([][]string, 0, len(count)+1)
firstLine := []string{"timeInterval", tool, "numberOfFiles"}
CSVDistribution = append(CSVDistribution, firstLine)
for _, interval := range distributionIntervals {
number := strconv.FormatInt(int64(count[interval]), 10)
row := []string{interval, tool, number}
CSVDistribution = append(CSVDistribution, row)
}
printDistributionInfo(count, tool)
return CSVDistribution, nil
}
func printDistributionInfo(count map[string]int, tool string) {
total := 0
for _, v := range count {
total += v
}
fmt.Println(tool, "files", total)
fmt.Println("Distribution")
for _, interval := range distributionIntervals {
fmt.Println("\t", interval, count[interval])
}
fmt.Println("Percentage")
for _, interval := range distributionIntervals {
p := (float64(count[interval]) / float64(total)) * 100.00
fmt.Printf("\t %s %f%%\n", interval, p)
}
fmt.Printf("\n\n")
}
func arrangeByTime(count map[string]int, num float64) {
switch {
case num > 1000.00 && num <= 10000.00:
count[distributionIntervals[0]]++
case num > 10000.00 && num <= 100000.00:
count[distributionIntervals[1]]++
case num > 100000.00 && num <= 1000000.00:
count[distributionIntervals[2]]++
case num > 1000000.00 && num <= 10000000.00:
count[distributionIntervals[3]]++
case num > 10000000.00 && num <= 100000000.00:
count[distributionIntervals[4]]++
}
}
func writeCSV(CSVData [][]string, outPath string) error {
out, err := os.Create(outPath)
if err != nil {
return err
}
w := csv.NewWriter(out)
w.WriteAll(CSVData)
if err := w.Error(); err != nil {
return err
}
return nil
}
type parse func(data []byte, tool string) ([][]string, error)
func generateCSV() {
bmFiles := []struct {
in string
out string
tool string
parse parse
}{
{in: enryTotalBench, out: enryTotalCSV, tool: "enry", parse: parseTotal},
{in: linguistTotalBench, out: linguistTotalCSV, tool: "linguist", parse: parseTotal},
{in: enrySamplesBench, out: enrySamplesCSV, tool: "enry", parse: parseSamples},
{in: linguistSamplesBench, out: linguistSamplesCSV, tool: "linguist", parse: parseSamples},
}
for _, bmFile := range bmFiles {
buf, err := ioutil.ReadFile(bmFile.in)
if err != nil {
log.Println(err)
continue
}
info, err := bmFile.parse(buf, bmFile.tool)
if err != nil {
log.Println(err)
continue
}
if err := writeCSV(info, filepath.Join(outDir, bmFile.out)); err != nil {
log.Println(err)
continue
}
}
}
func parseTotal(data []byte, tool string) ([][]string, error) {
const totalLine = "_TOTAL"
parsedInfo := map[string][]string{}
buf := bufio.NewScanner(bytes.NewReader(data))
for buf.Scan() {
line := buf.Text()
if strings.Contains(line, totalLine) {
split := strings.Fields(line)
row, err := getRow(split, tool)
if err != nil {
return nil, err
}
parsedInfo[row[0]] = row
}
}
if err := buf.Err(); err != nil {
return nil, err
}
firstLine := []string{"function", "tool", "iterations", "ns/op"}
return prepareInfoForCSV(parsedInfo, firstLine), nil
}
func getRow(line []string, tool string) ([]string, error) {
row := make([]string, 0, 3)
for _, function := range enryFunctions {
if strings.Contains(line[0], function) {
row = append(row, function)
break
}
}
row = append(row, tool)
iterations := line[1]
row = append(row, iterations)
average, err := getAverage(line)
if err != nil {
return nil, err
}
row = append(row, average)
return row, nil
}
func getAverage(line []string) (string, error) {
average := line[len(line)-1]
if !strings.HasSuffix(average, ")") {
return line[2], nil
}
totalTime := strings.Trim(average, "() ")
time, err := strconv.ParseFloat(totalTime, 64)
if err != nil {
return "", err
}
iterations := line[1]
i, err := strconv.ParseFloat(iterations, 64)
if err != nil {
return "", err
}
avg := (time * math.Pow10(9)) / i
return fmt.Sprintf("%d", int(avg)), nil
}
func prepareInfoForCSV(parsedInfo map[string][]string, firstLine []string) [][]string {
info := createInfoWithFirstLine(firstLine, len(parsedInfo))
for _, function := range enryFunctions {
info = append(info, parsedInfo[function])
}
return info
}
func createInfoWithFirstLine(firstLine []string, sliceLength int) (info [][]string) {
if len(firstLine) > 0 {
info = make([][]string, 0, sliceLength+1)
info = append(info, firstLine)
} else {
info = make([][]string, 0, sliceLength)
}
return
}
type enryFuncs map[string][]string
func newEnryFuncs() enryFuncs {
return enryFuncs{
getLanguageFunc: nil,
classifyFunc: nil,
modelineFunc: nil,
filenameFunc: nil,
shebangFunc: nil,
extensionFunc: nil,
contentFunc: nil,
}
}
func parseSamples(data []byte, tool string) ([][]string, error) {
const sampleLine = "SAMPLE_"
parsedInfo := map[string]enryFuncs{}
buf := bufio.NewScanner(bytes.NewReader(data))
for buf.Scan() {
line := buf.Text()
if strings.Contains(line, sampleLine) {
split := strings.Fields(line)
name := getSampleName(split[0])
if _, ok := parsedInfo[name]; !ok {
parsedInfo[name] = newEnryFuncs()
}
row := make([]string, 0, 4)
row = append(row, name)
r, err := getRow(split, tool)
if err != nil {
return nil, err
}
row = append(row, r...)
function := row[1]
parsedInfo[name][function] = row
}
}
if err := buf.Err(); err != nil {
return nil, err
}
firstLine := []string{"file", "function", "tool", "iterations", "ns/op"}
return prepareSamplesInfoForCSV(parsedInfo, firstLine), nil
}
func getSampleName(s string) string {
start := strings.Index(s, "SAMPLE_") + len("SAMPLE_")
suffix := fmt.Sprintf("-%d", runtime.GOMAXPROCS(-1))
name := strings.TrimSuffix(s[start:], suffix)
return name
}
func prepareSamplesInfoForCSV(parsedInfo map[string]enryFuncs, firstLine []string) [][]string {
info := createInfoWithFirstLine(firstLine, len(parsedInfo)*len(enryFunctions))
orderedKeys := sortKeys(parsedInfo)
for _, path := range orderedKeys {
sampleInfo := prepareInfoForCSV(parsedInfo[path], nil)
info = append(info, sampleInfo...)
}
return info
}
func sortKeys(parsedInfo map[string]enryFuncs) []string {
keys := make([]string, 0, len(parsedInfo))
for key := range parsedInfo {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}

21
benchmarks/plot-histogram.gp Executable file
View File

@ -0,0 +1,21 @@
#!/usr/bin/env gnuplot
set terminal png large font "arial,26" size 1920,1080
set output 'benchmarks/histogram/distribution.png'
set datafile separator comma
set key under
set style data histogram
set style histogram clustered gap 1 title offset 1,1
set style fill solid noborder
set boxwidth 0.95
set grid y
set bmargin 12
set autoscale
set title "Number of files per processing time"
plot newhistogram, 'benchmarks/csv/enry-distribution.csv' using 3:xtic(1) title "enry", 'benchmarks/csv/linguist-distribution.csv' using 3 title "linguist"
unset output

6
benchmarks/run-benchmarks.sh Executable file
View File

@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -e
mkdir -p benchmarks/output
go test -run NONE -bench=. -benchtime=120s -timeout=100h > benchmarks/output/enry_total.bench
benchmarks/linguist-total.rb 5 > benchmarks/output/linguist_total.bench

7
benchmarks/run.sh Executable file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -e
benchmarks/run-benchmarks.sh
make benchmarks-slow
benchmarks/parse.sh
benchmarks/plot-histogram.gp

9
benchmarks/soft-hard-info.txt Normal file
View File

@ -0,0 +1,9 @@
# Hardware and software used to run benchmarks
MacBookPro13,1
Darwin Kernel Version 16.7.0: Tue Jan 30 11:27:06 PST 2018; root:xnu-3789.73.11~1/RELEASE_X86_64 x86_64 i386
go version go1.10.3 darwin/amd64
ruby 2.4.1p111 (2017-03-22 revision 58053) [x86_64-darwin16]
github/linguist v7.1.3 commit: e761f9b013e5b61161481fcb898b59721ee40e3d
src-d/enry v1.6.7 commit: 3d356c70ae322f41048f74d01c5e8572f5898d34

107
classifier.go Normal file
View File

@ -0,0 +1,107 @@
package enry
import (
"math"
"sort"
"github.com/go-enry/go-enry/v2/internal/tokenizer"
)
// classifier is the interface in charge to detect the possible languages of the given content based on a set of
// candidates. Candidates is a map which can be used to assign weights to languages dynamically.
type classifier interface {
classify(content []byte, candidates map[string]float64) (languages []string)
}
type naiveBayes struct {
languagesLogProbabilities map[string]float64
tokensLogProbabilities map[string]map[string]float64
tokensTotal float64
}
type scoredLanguage struct {
language string
score float64
}
// classify returns a sorted slice of possible languages sorted by decreasing language's probability
func (c *naiveBayes) classify(content []byte, candidates map[string]float64) []string {
var languages map[string]float64
if len(candidates) == 0 {
languages = c.knownLangs()
} else {
languages = make(map[string]float64, len(candidates))
for candidate, weight := range candidates {
if lang, ok := GetLanguageByAlias(candidate); ok {
candidate = lang
}
languages[candidate] = weight
}
}
empty := len(content) == 0
scoredLangs := make([]*scoredLanguage, 0, len(languages))
var tokens []string
if !empty {
tokens = tokenizer.Tokenize(content)
}
for language := range languages {
score := c.languagesLogProbabilities[language]
if !empty {
score += c.tokensLogProbability(tokens, language)
}
scoredLangs = append(scoredLangs, &scoredLanguage{
language: language,
score: score,
})
}
return sortLanguagesByScore(scoredLangs)
}
func sortLanguagesByScore(scoredLangs []*scoredLanguage) []string {
sort.Stable(byScore(scoredLangs))
sortedLanguages := make([]string, 0, len(scoredLangs))
for _, scoredLang := range scoredLangs {
sortedLanguages = append(sortedLanguages, scoredLang.language)
}
return sortedLanguages
}
func (c *naiveBayes) knownLangs() map[string]float64 {
langs := make(map[string]float64, len(c.languagesLogProbabilities))
for lang := range c.languagesLogProbabilities {
langs[lang]++
}
return langs
}
func (c *naiveBayes) tokensLogProbability(tokens []string, language string) float64 {
var sum float64
for _, token := range tokens {
sum += c.tokenProbability(token, language)
}
return sum
}
func (c *naiveBayes) tokenProbability(token, language string) float64 {
tokenProb, ok := c.tokensLogProbabilities[language][token]
if !ok {
tokenProb = math.Log(1.000000 / c.tokensTotal)
}
return tokenProb
}
type byScore []*scoredLanguage
func (b byScore) Len() int { return len(b) }
func (b byScore) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b byScore) Less(i, j int) bool { return b[j].score < b[i].score }

385
cmd/enry/main.go Normal file
View File

@ -0,0 +1,385 @@
package main
import (
"bufio"
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
"sort"
"strings"
"github.com/go-enry/go-enry/v2"
"github.com/go-enry/go-enry/v2/data"
)
var (
version = "undefined"
build = "undefined"
commit = "undefined"
)
func main() {
flag.Usage = usage
breakdownFlag := flag.Bool("breakdown", false, "")
jsonFlag := flag.Bool("json", false, "")
showVersion := flag.Bool("version", false, "Show the enry version information")
allLangs := flag.Bool("all", false, "Show all files, including those identified as non-programming languages")
countMode := flag.String("mode", "byte", "the method used to count file size. Available options are: file, line and byte")
limitKB := flag.Int64("limit", 16*1024, "Analyse first N KB of the file (-1 means no limit)")
flag.Parse()
limit := (*limitKB) * 1024
if *showVersion {
fmt.Println(version)
return
}
root, err := filepath.Abs(flag.Arg(0))
if err != nil {
log.Fatal(err)
}
fileInfo, err := os.Stat(root)
if err != nil {
log.Fatal(err)
}
if fileInfo.Mode().IsRegular() {
err = printFileAnalysis(root, limit, *jsonFlag)
if err != nil {
fmt.Println(err)
}
return
}
out := make(map[string][]string, 0)
err = filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
if err != nil {
log.Println(err)
return filepath.SkipDir
}
if !f.Mode().IsDir() && !f.Mode().IsRegular() {
return nil
}
relativePath, err := filepath.Rel(root, path)
if err != nil {
log.Println(err)
return nil
}
if relativePath == "." {
return nil
}
if f.IsDir() {
relativePath = relativePath + "/"
}
if enry.IsVendor(relativePath) || enry.IsDotFile(relativePath) ||
enry.IsDocumentation(relativePath) || enry.IsConfiguration(relativePath) {
// TODO(bzz): skip enry.IsGeneratedPath() after https://github.com/src-d/enry/issues/213
if f.IsDir() {
return filepath.SkipDir
}
return nil
}
if f.IsDir() {
return nil
}
// TODO(bzz): provide API that mimics linguist CLI output for
// - running ByExtension & ByFilename
// - reading the file, if that did not work
// - GetLanguage([]Strategy)
content, err := readFile(path, limit)
if err != nil {
log.Println(err)
return nil
}
// TODO(bzz): skip enry.IsGeneratedContent() as well, after https://github.com/src-d/enry/issues/213
language := enry.GetLanguage(filepath.Base(path), content)
if language == enry.OtherLanguage {
return nil
}
// If we are not asked to display all, do as
// https://github.com/github/linguist/blob/bf95666fc15e49d556f2def4d0a85338423c25f3/lib/linguist/blob_helper.rb#L382
if !*allLangs &&
enry.GetLanguageType(language) != enry.Programming &&
enry.GetLanguageType(language) != enry.Markup {
return nil
}
out[language] = append(out[language], relativePath)
return nil
})
if err != nil {
log.Fatal(err)
}
var buf bytes.Buffer
switch {
case *jsonFlag && !*breakdownFlag:
printJson(out, &buf)
case *jsonFlag && *breakdownFlag:
printBreakDown(out, &buf)
case *breakdownFlag:
printPercents(root, out, &buf, *countMode)
buf.WriteByte('\n')
printBreakDown(out, &buf)
default:
printPercents(root, out, &buf, *countMode)
}
fmt.Print(buf.String())
}
func usage() {
fmt.Fprintf(
os.Stderr,
` %[1]s %[2]s build: %[3]s commit: %[4]s, based on linguist commit: %[5]s
%[1]s, A simple (and faster) implementation of github/linguist
usage: %[1]s [-mode=(file|line|byte)] [-prog] <path>
%[1]s [-mode=(file|line|byte)] [-prog] [-json] [-breakdown] <path>
%[1]s [-mode=(file|line|byte)] [-prog] [-json] [-breakdown]
%[1]s [-version]
`,
os.Args[0], version, build, commit, data.LinguistCommit[:7],
)
}
func printBreakDown(out map[string][]string, buff *bytes.Buffer) {
for name, language := range out {
fmt.Fprintln(buff, name)
for _, file := range language {
fmt.Fprintln(buff, file)
}
fmt.Fprintln(buff)
}
}
func printJson(out map[string][]string, buf *bytes.Buffer) {
json.NewEncoder(buf).Encode(out)
}
// filelistError represents a failed operation that took place across multiple files.
type filelistError []string
func (e filelistError) Error() string {
return fmt.Sprintf("Could not process the following files:\n%s", strings.Join(e, "\n"))
}
func printPercents(root string, fSummary map[string][]string, buff *bytes.Buffer, mode string) {
// Select the way we quantify 'amount' of code.
reducer := fileCountValues
switch mode {
case "file":
reducer = fileCountValues
case "line":
reducer = lineCountValues
case "byte":
reducer = byteCountValues
}
// Reduce the list of files to a quantity of file type.
var (
total float64
keys []string
unreadableFiles filelistError
fileValues = make(map[string]float64)
)
for fType, files := range fSummary {
val, err := reducer(root, files)
if err != nil {
unreadableFiles = append(unreadableFiles, err...)
}
fileValues[fType] = val
keys = append(keys, fType)
total += val
}
// Slice the keys by their quantity (file count, line count, byte size, etc.).
sort.Slice(keys, func(i, j int) bool {
return fileValues[keys[i]] > fileValues[keys[j]]
})
// Calculate and write percentages of each file type.
for _, fType := range keys {
val := fileValues[fType]
percent := val / total * 100.0
buff.WriteString(fmt.Sprintf("%.2f%%\t%s\n", percent, fType))
if unreadableFiles != nil {
buff.WriteString(fmt.Sprintf("\n%s", unreadableFiles.Error()))
}
}
}
func fileCountValues(_ string, files []string) (float64, filelistError) {
return float64(len(files)), nil
}
func lineCountValues(root string, files []string) (float64, filelistError) {
var filesErr filelistError
var t float64
for _, fName := range files {
l, _ := getLines(filepath.Join(root, fName), nil)
t += float64(l)
}
return t, filesErr
}
func byteCountValues(root string, files []string) (float64, filelistError) {
var filesErr filelistError
var t float64
for _, fName := range files {
f, err := os.Open(filepath.Join(root, fName))
if err != nil {
filesErr = append(filesErr, fName)
continue
}
fi, err := f.Stat()
f.Close()
if err != nil {
filesErr = append(filesErr, fName)
continue
}
t += float64(fi.Size())
}
return t, filesErr
}
func printFileAnalysis(file string, limit int64, isJSON bool) error {
data, err := readFile(file, limit)
if err != nil {
return err
}
isSample := limit > 0 && len(data) == int(limit)
full := data
if isSample {
// communicate to getLines that we don't have full contents
full = nil
}
totalLines, nonBlank := getLines(file, full)
// functions below can work on a sample
fileType := getFileType(file, data)
language := enry.GetLanguage(file, data)
mimeType := enry.GetMIMEType(file, language)
vendored := enry.IsVendor(file)
if isJSON {
return json.NewEncoder(os.Stdout).Encode(map[string]interface{}{
"filename": filepath.Base(file),
"lines": nonBlank,
"total_lines": totalLines,
"type": fileType,
"mime": mimeType,
"language": language,
"vendored": vendored,
})
}
fmt.Printf(
`%s: %d lines (%d sloc)
type: %s
mime_type: %s
language: %s
vendored: %t
`,
filepath.Base(file), totalLines, nonBlank, fileType, mimeType, language, vendored,
)
return nil
}
func readFile(path string, limit int64) ([]byte, error) {
if limit <= 0 {
return ioutil.ReadFile(path)
}
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
st, err := f.Stat()
if err != nil {
return nil, err
}
size := st.Size()
if limit > 0 && size > limit {
size = limit
}
buf := bytes.NewBuffer(nil)
buf.Grow(int(size))
_, err = io.Copy(buf, io.LimitReader(f, limit))
return buf.Bytes(), err
}
func getLines(file string, content []byte) (total, blank int) {
var r io.Reader
if content != nil {
r = bytes.NewReader(content)
} else {
// file not loaded to memory - stream it
f, err := os.Open(file)
if err != nil {
fmt.Println(err)
return
}
defer f.Close()
r = f
}
br := bufio.NewReader(r)
lastBlank := true
empty := true
for {
data, prefix, err := br.ReadLine()
if err == io.EOF {
break
} else if err != nil {
fmt.Println(err)
break
}
if prefix {
continue
}
empty = false
total++
lastBlank = len(data) == 0
if lastBlank {
blank++
}
}
if !empty && lastBlank {
total++
blank++
}
nonBlank := total - blank
return total, nonBlank
}
func getFileType(file string, content []byte) string {
switch {
case enry.IsImage(file):
return "Image"
case enry.IsBinary(content):
return "Binary"
default:
return "Text"
}
}

38
cmd/enry/main_test.go Normal file
View File

@ -0,0 +1,38 @@
package main
import (
"testing"
)
func TestGetLines(t *testing.T) {
tests := []struct {
content string
wantTotal int
wantNonBlank int
}{
// 0
{content: "This is one line", wantTotal: 1, wantNonBlank: 1},
// 1 Test no content
{content: "", wantTotal: 0, wantNonBlank: 0},
// 2 A single blank line
{content: "One blank line\n\nTwo nonblank lines", wantTotal: 3, wantNonBlank: 2},
// 3 Testing multiple blank lines in a row
{content: "\n\n", wantTotal: 3, wantNonBlank: 0},
// 4 '
{content: "\n\n\n\n", wantTotal: 5, wantNonBlank: 0},
// 5 Multiple blank lines content on ends
{content: "content\n\n\n\ncontent", wantTotal: 5, wantNonBlank: 2},
// 6 Content with blank lines on ends
{content: "\n\n\ncontent\n\n\n", wantTotal: 7, wantNonBlank: 1},
}
for i, test := range tests {
t.Run("", func(t *testing.T) {
gotTotal, gotNonBlank := getLines("", []byte(test.content))
if gotTotal != test.wantTotal || gotNonBlank != test.wantNonBlank {
t.Errorf("wrong line counts obtained for test case #%d:\n %7s, %7s\nGOT: %7d, %7d\nWANT: %7d, %7d\n", i, "TOTAL", "NON_BLANK",
gotTotal, gotNonBlank, test.wantTotal, test.wantNonBlank)
}
})
}
}

568
common.go Normal file
View File

@ -0,0 +1,568 @@
package enry
import (
"bufio"
"bytes"
"fmt"
"path"
"path/filepath"
"strings"
"github.com/go-enry/go-enry/v2/data"
"github.com/go-enry/go-enry/v2/regex"
)
// OtherLanguage is used as a zero value when a function can not return a specific language.
const OtherLanguage = ""
// Strategy type fix the signature for the functions that can be used as a strategy.
type Strategy func(filename string, content []byte, candidates []string) (languages []string)
// DefaultStrategies is a sequence of strategies used by GetLanguage to detect languages.
var DefaultStrategies = []Strategy{
GetLanguagesByModeline,
GetLanguagesByFilename,
GetLanguagesByShebang,
GetLanguagesByExtension,
GetLanguagesByXML,
GetLanguagesByManpage,
GetLanguagesByContent,
GetLanguagesByClassifier,
}
// defaultClassifier is a Naive Bayes classifier trained on Linguist samples.
var defaultClassifier classifier = &naiveBayes{
languagesLogProbabilities: data.LanguagesLogProbabilities,
tokensLogProbabilities: data.TokensLogProbabilities,
tokensTotal: data.TokensTotal,
}
// GetLanguage applies a sequence of strategies based on the given filename and content
// to find out the most probable language to return.
func GetLanguage(filename string, content []byte) (language string) {
languages := GetLanguages(filename, content)
return firstLanguage(languages)
}
func firstLanguage(languages []string) string {
for _, l := range languages {
if l != "" {
return l
}
}
return OtherLanguage
}
// GetLanguageByModeline returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByModeline(content []byte) (language string, safe bool) {
return getLanguageByStrategy(GetLanguagesByModeline, "", content, nil)
}
// GetLanguageByEmacsModeline returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByEmacsModeline(content []byte) (language string, safe bool) {
return getLanguageByStrategy(GetLanguagesByEmacsModeline, "", content, nil)
}
// GetLanguageByVimModeline returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByVimModeline(content []byte) (language string, safe bool) {
return getLanguageByStrategy(GetLanguagesByVimModeline, "", content, nil)
}
// GetLanguageByFilename returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByFilename(filename string) (language string, safe bool) {
return getLanguageByStrategy(GetLanguagesByFilename, filename, nil, nil)
}
// GetLanguageByShebang returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByShebang(content []byte) (language string, safe bool) {
return getLanguageByStrategy(GetLanguagesByShebang, "", content, nil)
}
// GetLanguageByExtension returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByExtension(filename string) (language string, safe bool) {
return getLanguageByStrategy(GetLanguagesByExtension, filename, nil, nil)
}
// GetLanguageByContent returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByContent(filename string, content []byte) (language string, safe bool) {
return getLanguageByStrategy(GetLanguagesByContent, filename, content, nil)
}
// GetLanguageByClassifier returns the most probably language detected for the given content. It uses
// defaultClassifier, if no candidates are provided it returns OtherLanguage.
func GetLanguageByClassifier(content []byte, candidates []string) (language string, safe bool) {
return getLanguageByStrategy(GetLanguagesByClassifier, "", content, candidates)
}
func getLanguageByStrategy(strategy Strategy, filename string, content []byte, candidates []string) (string, bool) {
languages := strategy(filename, content, candidates)
return getFirstLanguageAndSafe(languages)
}
func getFirstLanguageAndSafe(languages []string) (language string, safe bool) {
language = firstLanguage(languages)
safe = len(languages) == 1
return
}
// GetLanguages applies a sequence of strategies based on the given filename and content
// to find out the most probable languages to return.
//
// If it finds a strategy that produces a single result, it will be returned;
// otherise the last strategy that returned multiple results will be returned.
// If the content is binary, no results will be returned. This matches the
// behavior of Linguist.detect: https://github.com/github/linguist/blob/aad49acc0624c70d654a8dce447887dbbc713c7a/lib/linguist.rb#L14-L49
//
// At least one of arguments should be set. If content is missing, language detection will be based on the filename.
// The function won't read the file, given an empty content.
func GetLanguages(filename string, content []byte) []string {
if IsBinary(content) {
return nil
}
var languages []string
for _, strategy := range DefaultStrategies {
candidates := strategy(filename, content, languages)
// No candidates, continue to next strategy without updating languages
if len(candidates) == 0 {
continue
}
// Only one candidate match, return it
if len(candidates) == 1 {
return candidates
}
// Save the candidates from this strategy to pass onto to the next strategy, like Linguist
languages = candidates
}
return languages
}
// GetLanguagesByModeline returns a slice of possible languages for the given content.
// It complies with the signature to be a Strategy type.
func GetLanguagesByModeline(_ string, content []byte, candidates []string) []string {
headFoot := getHeaderAndFooter(content)
var languages []string
for _, getLang := range modelinesFunc {
languages = getLang("", headFoot, candidates)
if len(languages) > 0 {
break
}
}
return languages
}
var modelinesFunc = []Strategy{
GetLanguagesByEmacsModeline,
GetLanguagesByVimModeline,
}
func getHeaderAndFooter(content []byte) []byte {
const searchScope = 5
if len(content) == 0 {
return content
}
if bytes.Count(content, []byte("\n")) < 2*searchScope {
return content
}
header := headScope(content, searchScope)
footer := footScope(content, searchScope)
headerAndFooter := make([]byte, 0, len(content[:header])+len(content[footer:]))
headerAndFooter = append(headerAndFooter, content[:header]...)
headerAndFooter = append(headerAndFooter, content[footer:]...)
return headerAndFooter
}
func headScope(content []byte, scope int) (index int) {
for i := 0; i < scope; i++ {
eol := bytes.IndexAny(content, "\n")
content = content[eol+1:]
index += eol
}
return index + scope - 1
}
func footScope(content []byte, scope int) (index int) {
for i := 0; i < scope; i++ {
index = bytes.LastIndexAny(content, "\n")
content = content[:index]
}
return index + 1
}
var (
reEmacsModeline = regex.MustCompile(`.*-\*-\s*(.+?)\s*-\*-.*(?m:$)`)
reEmacsLang = regex.MustCompile(`.*(?i:mode)\s*:\s*([^\s;]+)\s*;*.*`)
reVimModeline = regex.MustCompile(`(?:(?m:\s|^)vi(?:m[<=>]?\d+|m)?|[\t\x20]*ex)\s*[:]\s*(.*)(?m:$)`)
reVimLang = regex.MustCompile(`(?i:filetype|ft|syntax)\s*=(\w+)(?:\s|:|$)`)
)
// GetLanguagesByEmacsModeline returns a slice of possible languages for the given content.
// It complies with the signature to be a Strategy type.
func GetLanguagesByEmacsModeline(_ string, content []byte, _ []string) []string {
matched := reEmacsModeline.FindAllSubmatch(content, -1)
if matched == nil {
return nil
}
// only take the last matched line, discard previous lines
lastLineMatched := matched[len(matched)-1][1]
matchedAlias := reEmacsLang.FindSubmatch(lastLineMatched)
var alias string
if matchedAlias != nil {
alias = string(matchedAlias[1])
} else {
alias = string(lastLineMatched)
}
language, ok := GetLanguageByAlias(alias)
if !ok {
return nil
}
return []string{language}
}
// GetLanguagesByVimModeline returns a slice of possible languages for the given content.
// It complies with the signature to be a Strategy type.
func GetLanguagesByVimModeline(_ string, content []byte, _ []string) []string {
matched := reVimModeline.FindAllSubmatch(content, -1)
if matched == nil {
return nil
}
// only take the last matched line, discard previous lines
lastLineMatched := matched[len(matched)-1][1]
matchedAlias := reVimLang.FindAllSubmatch(lastLineMatched, -1)
if matchedAlias == nil {
return nil
}
alias := string(matchedAlias[0][1])
if len(matchedAlias) > 1 {
// cases:
// matchedAlias = [["syntax=ruby " "ruby"] ["ft=python " "python"] ["filetype=perl " "perl"]] returns OtherLanguage;
// matchedAlias = [["syntax=python " "python"] ["ft=python " "python"] ["filetype=python " "python"]] returns "Python";
for _, match := range matchedAlias {
otherAlias := string(match[1])
if otherAlias != alias {
return nil
}
}
}
language, ok := GetLanguageByAlias(alias)
if !ok {
return nil
}
return []string{language}
}
// GetLanguagesByFilename returns a slice of possible languages for the given filename.
// It complies with the signature to be a Strategy type.
func GetLanguagesByFilename(filename string, _ []byte, _ []string) []string {
if filename == "" {
return nil
}
return data.LanguagesByFilename[filepath.Base(filename)]
}
// GetLanguagesByShebang returns a slice of possible languages for the given content.
// It complies with the signature to be a Strategy type.
func GetLanguagesByShebang(_ string, content []byte, _ []string) (languages []string) {
interpreter := getInterpreter(content)
return data.LanguagesByInterpreter[interpreter]
}
var (
shebangExecHack = regex.MustCompile(`exec (\w+).+\$0.+\$@`)
pythonVersion = regex.MustCompile(`python\d\.\d+`)
envOptArgs = regex.MustCompile(`-[i0uCSv]*|--\S+`)
envVarArgs = regex.MustCompile(`\S+=\S+`)
)
func getInterpreter(data []byte) string {
line := getFirstLine(data)
if !hasShebang(line) {
return ""
}
// skip shebang
line = bytes.TrimSpace(line[2:])
splitted := bytes.Fields(line)
if len(splitted) == 0 {
return ""
}
// Extract interpreter name from path. Use path.Base because
// shebang on Cygwin/Windows still use a forward slash
interpreter := path.Base(string(splitted[0]))
// #!/usr/bin/env [...]
if interpreter == "env" {
if len(splitted) == 1 {
// /usr/bin/env with no arguments
return ""
}
for len(splitted) > 2 {
if envOptArgs.Match(splitted[1]) || envVarArgs.Match(splitted[1]) {
splitted = append(splitted[:1], splitted[2:]...)
continue
}
break
}
interpreter = path.Base(string(splitted[1]))
}
if interpreter == "sh" {
interpreter = lookForMultilineExec(data)
}
if pythonVersion.MatchString(interpreter) {
interpreter = interpreter[:strings.Index(interpreter, `.`)]
}
// If osascript is called with argument -l it could be different language so do not relay on it
// To match linguist behaviour, see ref https://github.com/github/linguist/blob/d95bae794576ab0ef2fcb41a39eb61ea5302c5b5/lib/linguist/shebang.rb#L63
if interpreter == "osascript" && bytes.Contains(line, []byte("-l")) {
interpreter = ""
}
return interpreter
}
func getFirstLines(content []byte, count int) []byte {
nlpos := -1
for ; count > 0; count-- {
pos := bytes.IndexByte(content[nlpos+1:], '\n')
if pos < 0 {
return content
}
nlpos += pos + 1
}
return content[:nlpos]
}
func getFirstLine(content []byte) []byte {
return getFirstLines(content, 1)
}
func hasShebang(line []byte) bool {
const shebang = `#!`
prefix := []byte(shebang)
return bytes.HasPrefix(line, prefix)
}
func lookForMultilineExec(data []byte) string {
const magicNumOfLines = 5
interpreter := "sh"
buf := bufio.NewScanner(bytes.NewReader(data))
for i := 0; i < magicNumOfLines && buf.Scan(); i++ {
line := buf.Bytes()
if shebangExecHack.Match(line) {
interpreter = shebangExecHack.FindStringSubmatch(string(line))[1]
break
}
}
if err := buf.Err(); err != nil {
return interpreter
}
return interpreter
}
// GetLanguagesByExtension returns a slice of possible languages for the given filename.
// It complies with the signature to be a Strategy type.
func GetLanguagesByExtension(filename string, _ []byte, _ []string) []string {
if !strings.Contains(filename, ".") {
return nil
}
filename = strings.ToLower(filename)
dots := getDotIndexes(filename)
for _, dot := range dots {
ext := filename[dot:]
languages, ok := data.LanguagesByExtension[ext]
if ok {
return languages
}
}
return nil
}
var (
manpageExtension = regex.MustCompile(`\.(?:[1-9](?:[a-z_]+[a-z_0-9]*)?|0p|n|man|mdoc)(?:\.in)?$`)
)
// GetLanguagesByManpage returns a slice of possible manpage languages for the given filename.
// It complies with the signature to be a Strategy type.
func GetLanguagesByManpage(filename string, _ []byte, _ []string) []string {
filename = strings.ToLower(filename)
// Check if matches Roff man page filenames
if manpageExtension.Match([]byte(filename)) {
return []string{
"Roff Manpage",
"Roff",
}
}
return nil
}
var (
xmlHeader = regex.MustCompile(`<?xml version=`)
)
// GetLanguagesByXML returns a slice of possible XML language for the given filename.
// It complies with the signature to be a Strategy type.
func GetLanguagesByXML(_ string, content []byte, candidates []string) []string {
if len(candidates) > 0 {
return candidates
}
header := getFirstLines(content, 2)
// Check if contains XML header
if xmlHeader.Match(header) {
return []string{
"XML",
}
}
return nil
}
func getDotIndexes(filename string) []int {
dots := make([]int, 0, 2)
for i, letter := range filename {
if letter == rune('.') {
dots = append(dots, i)
}
}
return dots
}
// GetLanguagesByContent returns a slice of languages for the given content.
// It is a Strategy that uses content-based regexp heuristics and a filename extension.
func GetLanguagesByContent(filename string, content []byte, _ []string) []string {
if filename == "" {
return nil
}
ext := strings.ToLower(filepath.Ext(filename))
heuristic, ok := data.ContentHeuristics[ext]
if !ok {
return nil
}
return heuristic.Match(content)
}
// GetLanguagesByClassifier returns a sorted slice of possible languages ordered by
// decreasing language's probability. If there are not candidates it returns nil.
// It is a Strategy that uses a pre-trained defaultClassifier.
func GetLanguagesByClassifier(filename string, content []byte, candidates []string) (languages []string) {
if len(candidates) == 0 {
return nil
}
return getLanguagesBySpecificClassifier(content, candidates, defaultClassifier)
}
// getLanguagesBySpecificClassifier returns a slice of possible languages. It takes in a Classifier to be used.
func getLanguagesBySpecificClassifier(content []byte, candidates []string, classifier classifier) (languages []string) {
mapCandidates := make(map[string]float64)
for _, candidate := range candidates {
mapCandidates[candidate]++
}
return classifier.classify(content, mapCandidates)
}
// GetLanguageExtensions returns all extensions associated with the given language.
func GetLanguageExtensions(language string) []string {
return data.ExtensionsByLanguage[language]
}
// GetLanguageType returns the type of the given language.
func GetLanguageType(language string) (langType Type) {
intType, ok := data.LanguagesType[language]
langType = Type(intType)
if !ok {
langType = Unknown
}
return langType
}
// GetLanguageGroup returns language group or empty string if language does not have group.
func GetLanguageGroup(language string) string {
if group, ok := data.LanguagesGroup[language]; ok {
return group
}
return ""
}
// GetLanguageByAlias returns either the language related to the given alias and ok set to true
// or Otherlanguage and ok set to false if the alias is not recognized.
func GetLanguageByAlias(alias string) (lang string, ok bool) {
lang, ok = data.LanguageByAlias(alias)
if !ok {
lang = OtherLanguage
}
return
}
// GetLanguageID returns the ID for the language. IDs are assigned by GitHub.
// The input must be the canonical language name. Aliases are not supported.
//
// NOTE: The zero value (0) is a valid language ID, so this API mimics the Go
// map API. Use the second return value to check if the language was found.
func GetLanguageID(language string) (int, bool) {
id, ok := data.IDByLanguage[language]
return id, ok
}
// GetLanguageInfo returns the LanguageInfo for a given language name, or an error if not found.
func GetLanguageInfo(language string) (data.LanguageInfo, error) {
id, ok := GetLanguageID(language)
if !ok {
return data.LanguageInfo{}, fmt.Errorf("language %q not found", language)
}
return GetLanguageInfoByID(id)
}
// GetLanguageInfoByID returns the LanguageInfo for a given language ID, or an error if not found.
func GetLanguageInfoByID(id int) (data.LanguageInfo, error) {
if info, ok := data.LanguageInfoByID[id]; ok {
return info, nil
}
return data.LanguageInfo{}, fmt.Errorf("language %q not found", id)
}

655
common_test.go Normal file
View File

@ -0,0 +1,655 @@
package enry
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"github.com/go-enry/go-enry/v2/data"
"github.com/go-enry/go-enry/v2/internal/tests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
const linguistURL = "https://github.com/github/linguist.git"
const linguistClonedEnvVar = "ENRY_TEST_REPO"
func maybeCloneLinguist() (string, bool, error) {
return tests.MaybeCloneLinguist(linguistClonedEnvVar, linguistURL, data.LinguistCommit)
}
type enryBaseTestSuite struct {
suite.Suite
tmpLinguistDir string
isCleanupNeeded bool
samplesDir string
testFixturesDir string
}
func (s *enryBaseTestSuite) SetupSuite() {
var err error
s.tmpLinguistDir, s.isCleanupNeeded, err = maybeCloneLinguist()
require.NoError(s.T(), err)
s.samplesDir = filepath.Join(s.tmpLinguistDir, "samples")
s.testFixturesDir = filepath.Join(s.tmpLinguistDir, "test", "fixtures")
}
func (s *enryBaseTestSuite) TearDownSuite() {
if s.isCleanupNeeded {
err := os.RemoveAll(s.tmpLinguistDir)
require.NoError(s.T(), err)
}
}
type enryTestSuite struct {
enryBaseTestSuite
}
func Test_EnryTestSuite(t *testing.T) {
suite.Run(t, new(enryTestSuite))
}
func (s *enryTestSuite) TestRegexpEdgeCases() {
var regexpEdgeCases = []struct {
lang string
filename string
}{
{lang: "ActionScript", filename: "FooBar.as"},
{lang: "Forth", filename: "asm.fr"},
{lang: "X PixMap", filename: "cc-public_domain_mark_white.pm"},
//{lang: "SQL", filename: "drop_stuff.sql"}, // https://github.com/src-d/enry/issues/194
{lang: "Fstar", filename: "Hacl.Spec.Bignum.Fmul.fst"},
{lang: "C++", filename: "Types.h"},
}
for _, r := range regexpEdgeCases {
filename := filepath.Join(s.tmpLinguistDir, "samples", r.lang, r.filename)
content, err := ioutil.ReadFile(filename)
require.NoError(s.T(), err)
lang := GetLanguage(r.filename, content)
s.T().Logf("File:%s, lang:%s", filename, lang)
expLang, _ := data.LanguageByAlias(r.lang)
require.EqualValues(s.T(), expLang, lang)
}
}
func (s *enryTestSuite) TestGetLanguage() {
tests := []struct {
name string
filename string
content []byte
expected string
safe bool
}{
{name: "TestGetLanguage_0", filename: "foo.h", content: []byte{}, expected: "C"},
{name: "TestGetLanguage_1", filename: "foo.py", content: []byte{}, expected: "Python"},
{name: "TestGetLanguage_2", filename: "foo.m", content: []byte(":- module"), expected: "Mercury"},
{name: "TestGetLanguage_3", filename: "foo.m", content: nil, expected: "MATLAB"},
{name: "TestGetLanguage_4", filename: "foo.mo", content: []byte{0xDE, 0x12, 0x04, 0x95, 0x00, 0x00, 0x00, 0x00}, expected: OtherLanguage},
{name: "TestGetLanguage_5", filename: "", content: nil, expected: OtherLanguage},
}
for _, test := range tests {
language := GetLanguage(test.filename, test.content)
assert.Equal(s.T(), test.expected, language, fmt.Sprintf("%v: %v, expected: %v", test.name, language, test.expected))
}
}
func (s *enryTestSuite) TestGetLanguages() {
tests := []struct {
name string
filename string
content []byte
expected []string
}{
// With no content or filename, no language can be detected
{name: "TestGetLanguages_0", filename: "", content: []byte{}, expected: nil},
// The strategy that will match is GetLanguagesByExtension. Lacking content, it will return those results.
{name: "TestGetLanguages_1", filename: "foo.h", content: []byte{}, expected: []string{"C"}},
// GetLanguagesByExtension will return an unambiguous match when there is a single result.
{name: "TestGetLanguages_2", filename: "foo.groovy", content: []byte{}, expected: []string{"Groovy"}},
// GetLanguagesByExtension will return "Rust", "RenderScript" for .rs,
// then GetLanguagesByContent will take the first rule that matches (in this case Rust)
{name: "TestGetLanguages_3", filename: "foo.rs", content: []byte("use \n#include"), expected: []string{"Rust"}},
// .. and in this case, RenderScript (no content that matches a Rust regex can be included, because it runs first.)
{name: "TestGetLanguages_4", filename: "foo.rs", content: []byte("#include"), expected: []string{"RenderScript"}},
// GetLanguagesByExtension will return "AMPL", "Linux Kernel Module", "Modula-2", "XML",
// then GetLanguagesByContent will ALWAYS return Linux Kernel Module and AMPL when there is no content,
// and no further classifier can do anything without content
{name: "TestGetLanguages_5", filename: "foo.mod", content: []byte{}, expected: []string{"Linux Kernel Module", "AMPL"}},
// ...with some AMPL tokens, the DefaultClassifier will pick AMPL as the most likely language.
{name: "TestGetLanguages_6", filename: "foo.mod", content: []byte("BEAMS ROWS - TotalWeight"), expected: []string{"AMPL", "Linux Kernel Module"}},
}
for _, test := range tests {
languages := GetLanguages(test.filename, test.content)
assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: %v, expected: %v", test.name, languages, test.expected))
}
}
func (s *enryTestSuite) TestGetLanguagesByModelineLinguist() {
var modelinesDir = filepath.Join(s.tmpLinguistDir, "test", "fixtures", "Data", "Modelines")
tests := []struct {
name string
filename string
candidates []string
expected []string
}{
// Emacs
{name: "TestGetLanguagesByModelineLinguist_1", filename: filepath.Join(modelinesDir, "example_smalltalk.md"), expected: []string{"Smalltalk"}},
{name: "TestGetLanguagesByModelineLinguist_2", filename: filepath.Join(modelinesDir, "fundamentalEmacs.c"), expected: []string{"Text"}},
{name: "TestGetLanguagesByModelineLinguist_3", filename: filepath.Join(modelinesDir, "iamphp.inc"), expected: []string{"PHP"}},
{name: "TestGetLanguagesByModelineLinguist_4", filename: filepath.Join(modelinesDir, "seeplusplusEmacs1"), expected: []string{"C++"}},
{name: "TestGetLanguagesByModelineLinguist_5", filename: filepath.Join(modelinesDir, "seeplusplusEmacs2"), expected: []string{"C++"}},
{name: "TestGetLanguagesByModelineLinguist_6", filename: filepath.Join(modelinesDir, "seeplusplusEmacs3"), expected: []string{"C++"}},
{name: "TestGetLanguagesByModelineLinguist_7", filename: filepath.Join(modelinesDir, "seeplusplusEmacs4"), expected: []string{"C++"}},
{name: "TestGetLanguagesByModelineLinguist_8", filename: filepath.Join(modelinesDir, "seeplusplusEmacs5"), expected: []string{"C++"}},
{name: "TestGetLanguagesByModelineLinguist_9", filename: filepath.Join(modelinesDir, "seeplusplusEmacs6"), expected: []string{"C++"}},
{name: "TestGetLanguagesByModelineLinguist_10", filename: filepath.Join(modelinesDir, "seeplusplusEmacs7"), expected: []string{"C++"}},
{name: "TestGetLanguagesByModelineLinguist_11", filename: filepath.Join(modelinesDir, "seeplusplusEmacs9"), expected: []string{"C++"}},
{name: "TestGetLanguagesByModelineLinguist_12", filename: filepath.Join(modelinesDir, "seeplusplusEmacs10"), expected: []string{"C++"}},
{name: "TestGetLanguagesByModelineLinguist_13", filename: filepath.Join(modelinesDir, "seeplusplusEmacs11"), expected: []string{"C++"}},
{name: "TestGetLanguagesByModelineLinguist_14", filename: filepath.Join(modelinesDir, "seeplusplusEmacs12"), expected: []string{"C++"}},
// Vim
{name: "TestGetLanguagesByModelineLinguist_15", filename: filepath.Join(modelinesDir, "seeplusplus"), expected: []string{"C++"}},
{name: "TestGetLanguagesByModelineLinguist_16", filename: filepath.Join(modelinesDir, "iamjs.pl"), expected: []string{"JavaScript"}},
{name: "TestGetLanguagesByModelineLinguist_17", filename: filepath.Join(modelinesDir, "iamjs2.pl"), expected: []string{"JavaScript"}},
{name: "TestGetLanguagesByModelineLinguist_18", filename: filepath.Join(modelinesDir, "not_perl.pl"), expected: []string{"Prolog"}},
{name: "TestGetLanguagesByModelineLinguist_19", filename: filepath.Join(modelinesDir, "ruby"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByModelineLinguist_20", filename: filepath.Join(modelinesDir, "ruby2"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByModelineLinguist_21", filename: filepath.Join(modelinesDir, "ruby3"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByModelineLinguist_22", filename: filepath.Join(modelinesDir, "ruby4"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByModelineLinguist_23", filename: filepath.Join(modelinesDir, "ruby5"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByModelineLinguist_24", filename: filepath.Join(modelinesDir, "ruby6"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByModelineLinguist_25", filename: filepath.Join(modelinesDir, "ruby7"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByModelineLinguist_26", filename: filepath.Join(modelinesDir, "ruby8"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByModelineLinguist_27", filename: filepath.Join(modelinesDir, "ruby9"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByModelineLinguist_28", filename: filepath.Join(modelinesDir, "ruby10"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByModelineLinguist_29", filename: filepath.Join(modelinesDir, "ruby11"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByModelineLinguist_30", filename: filepath.Join(modelinesDir, "ruby12"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByModelineLinguist_31", filename: filepath.Join(s.samplesDir, "C++/runtime-compiler.cc"), expected: nil},
{name: "TestGetLanguagesByModelineLinguist_32", filename: "", expected: nil},
}
for _, test := range tests {
var content []byte
var err error
if test.filename != "" {
content, err = ioutil.ReadFile(test.filename)
assert.NoError(s.T(), err)
}
languages := GetLanguagesByModeline(test.filename, content, test.candidates)
assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected))
}
}
func (s *enryTestSuite) TestGetLanguagesByModeline() {
const (
wrongVim = `# vim: set syntax=ruby ft =python filetype=perl :`
rightVim = `/* vim: set syntax=python ft =python filetype=python */`
noLangVim = `/* vim: set shiftwidth=4 softtabstop=0 cindent cinoptions={1s: */`
)
tests := []struct {
name string
filename string
content []byte
candidates []string
expected []string
}{
{name: "TestGetLanguagesByModeline_1", content: []byte(wrongVim), expected: nil},
{name: "TestGetLanguagesByModeline_2", content: []byte(rightVim), expected: []string{"Python"}},
{name: "TestGetLanguagesByModeline_3", content: []byte(noLangVim), expected: nil},
{name: "TestGetLanguagesByModeline_4", content: nil, expected: nil},
{name: "TestGetLanguagesByModeline_5", content: []byte{}, expected: nil},
}
for _, test := range tests {
languages := GetLanguagesByModeline(test.filename, test.content, test.candidates)
assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected))
}
}
func (s *enryTestSuite) TestGetLanguagesByFilename() {
tests := []struct {
name string
filename string
content []byte
candidates []string
expected []string
}{
{name: "TestGetLanguagesByFilename_1", filename: "unknown.interpreter", expected: nil},
{name: "TestGetLanguagesByFilename_2", filename: ".bashrc", expected: []string{"Shell"}},
{name: "TestGetLanguagesByFilename_3", filename: "Dockerfile", expected: []string{"Dockerfile"}},
{name: "TestGetLanguagesByFilename_4", filename: "Makefile.frag", expected: []string{"Makefile"}},
{name: "TestGetLanguagesByFilename_5", filename: "makefile", expected: []string{"Makefile"}},
{name: "TestGetLanguagesByFilename_6", filename: "Vagrantfile", expected: []string{"Ruby"}},
{name: "TestGetLanguagesByFilename_7", filename: "_vimrc", expected: []string{"Vim Script"}},
{name: "TestGetLanguagesByFilename_8", filename: "pom.xml", expected: []string{"Maven POM"}},
{name: "TestGetLanguagesByFilename_9", filename: "", expected: nil},
{name: "TestGetLanguagesByFilename_10", filename: "hi.py", expected: nil}, // LanguagesByFilename is only for the well-known file names (.gitignore, etc)
}
for _, test := range tests {
languages := GetLanguagesByFilename(test.filename, test.content, test.candidates)
assert.Equal(s.T(), len(test.expected), len(languages), fmt.Sprintf("%v: number of languages = %v, expected: %v", test.name, len(languages), len(test.expected)))
for i := range languages { // case-insensitive name comparison
assert.True(s.T(), strings.EqualFold(test.expected[i], languages[i]), fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected))
}
}
}
func (s *enryTestSuite) TestGetLanguagesByShebang() {
const (
multilineExecHack = `#!/bin/sh
# Next line is comment in Tcl, but not in sh... \
exec tclsh "$0" ${1+"$@"}`
multilineNoExecHack = `#!/bin/sh
#<<<#
echo "A shell script in a zkl program ($0)"
echo "Now run zkl <this file> with Hello World as args"
zkl $0 Hello World!
exit
#<<<#
println("The shell script says ",vm.arglist.concat(" "));`
)
tests := []struct {
name string
filename string
content []byte
candidates []string
expected []string
}{
{name: "TestGetLanguagesByShebang_1", content: []byte(`#!/unknown/interpreter`), expected: nil},
{name: "TestGetLanguagesByShebang_2", content: []byte(`no shebang`), expected: nil},
{name: "TestGetLanguagesByShebang_3", content: []byte(`#!/usr/bin/env`), expected: nil},
{name: "TestGetLanguagesByShebang_4", content: []byte(`#!/usr/bin/python -tt`), expected: []string{"Python"}},
{name: "TestGetLanguagesByShebang_5", content: []byte(`#!/usr/bin/env python2.6`), expected: []string{"Python"}},
{name: "TestGetLanguagesByShebang_6", content: []byte(`#!/usr/bin/env perl`), expected: []string{"Perl", "Pod"}},
{name: "TestGetLanguagesByShebang_7", content: []byte(`#! /bin/sh`), expected: []string{"Shell"}},
{name: "TestGetLanguagesByShebang_8", content: []byte(`#!bash`), expected: []string{"Shell"}},
{name: "TestGetLanguagesByShebang_9", content: []byte(multilineExecHack), expected: []string{"Tcl"}},
{name: "TestGetLanguagesByShebang_10", content: []byte(multilineNoExecHack), expected: []string{"Shell"}},
{name: "TestGetLanguagesByShebang_11", content: []byte(`#!/envinpath/python`), expected: []string{"Python"}},
{name: "TestGetLanguagesByShebang_12", content: []byte(""), expected: nil},
{name: "TestGetLanguagesByShebang_13", content: []byte("foo"), expected: nil},
{name: "TestGetLanguagesByShebang_14", content: []byte("#bar"), expected: nil},
{name: "TestGetLanguagesByShebang_15", content: []byte("#baz"), expected: nil},
{name: "TestGetLanguagesByShebang_16", content: []byte("///"), expected: nil},
{name: "TestGetLanguagesByShebang_17", content: []byte("\n\n\n\n\n"), expected: nil},
{name: "TestGetLanguagesByShebang_18", content: []byte(" #!/usr/sbin/ruby"), expected: nil},
{name: "TestGetLanguagesByShebang_19", content: []byte("\n#!/usr/sbin/ruby"), expected: nil},
{name: "TestGetLanguagesByShebang_20", content: []byte("#!"), expected: nil},
{name: "TestGetLanguagesByShebang_21", content: []byte("#! "), expected: nil},
{name: "TestGetLanguagesByShebang_22", content: []byte("#!/usr/bin/env"), expected: nil},
{name: "TestGetLanguagesByShebang_23", content: []byte("#!/usr/bin/env osascript -l JavaScript"), expected: nil},
{name: "TestGetLanguagesByShebang_24", content: []byte("#!/usr/bin/env osascript -l AppleScript"), expected: nil},
{name: "TestGetLanguagesByShebang_25", content: []byte("#!/usr/bin/env osascript -l foobar"), expected: nil},
{name: "TestGetLanguagesByShebang_26", content: []byte("#!/usr/bin/osascript -l JavaScript"), expected: nil},
{name: "TestGetLanguagesByShebang_27", content: []byte("#!/usr/bin/osascript -l foobar"), expected: nil},
{name: "TestGetLanguagesByShebang_28", content: []byte("#!/usr/sbin/ruby\n# bar"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByShebang_29", content: []byte("#!/usr/bin/ruby\n# foo"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByShebang_30", content: []byte("#!/usr/sbin/ruby"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByShebang_31", content: []byte("#!/usr/sbin/ruby foo bar baz\n"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByShebang_32", content: []byte("#!/usr/bin/env Rscript\n# example R script\n#\n"), expected: []string{"R"}},
{name: "TestGetLanguagesByShebang_33", content: []byte("#!/usr/bin/env ruby\n# baz"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByShebang_34", content: []byte("#!/usr/bin/bash\n"), expected: []string{"Shell"}},
{name: "TestGetLanguagesByShebang_35", content: []byte("#!/bin/sh"), expected: []string{"Shell"}},
{name: "TestGetLanguagesByShebang_36", content: []byte("#!/bin/python\n# foo\n# bar\n# baz"), expected: []string{"Python"}},
{name: "TestGetLanguagesByShebang_37", content: []byte("#!/usr/bin/python2.7\n\n\n\n"), expected: []string{"Python"}},
{name: "TestGetLanguagesByShebang_38", content: []byte("#!/usr/bin/python3\n\n\n\n"), expected: []string{"Python"}},
{name: "TestGetLanguagesByShebang_39", content: []byte("#!/usr/bin/sbcl --script\n\n"), expected: []string{"Common Lisp"}},
{name: "TestGetLanguagesByShebang_40", content: []byte("#! perl"), expected: []string{"Perl", "Pod"}},
{name: "TestGetLanguagesByShebang_41", content: []byte("#!/bin/sh\n\n\nexec ruby $0 $@"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByShebang_42", content: []byte("#! /usr/bin/env A=003 B=149 C=150 D=xzd E=base64 F=tar G=gz H=head I=tail sh"), expected: []string{"Shell"}},
{name: "TestGetLanguagesByShebang_43", content: []byte("#!/usr/bin/env foo=bar bar=foo python -cos=__import__(\"os\");"), expected: []string{"Python"}},
{name: "TestGetLanguagesByShebang_44", content: []byte("#!/usr/bin/env osascript"), expected: []string{"AppleScript"}},
{name: "TestGetLanguagesByShebang_45", content: []byte("#!/usr/bin/osascript"), expected: []string{"AppleScript"}},
{name: "TestGetLanguagesByShebang_46", content: []byte("#!/usr/bin/env -vS ruby -wKU\nputs ?t+?e+?s+?t"), expected: []string{"Ruby"}},
{name: "TestGetLanguagesByShebang_47", content: []byte("#!/usr/bin/env --split-string sed -f\ny/a/A/"), expected: []string{"sed"}},
{name: "TestGetLanguagesByShebang_48", content: []byte("#!/usr/bin/env -S GH_TOKEN=ghp_*** deno run --allow-net\nconsole.log(1);"), expected: []string{"TypeScript"}},
}
for _, test := range tests {
languages := GetLanguagesByShebang(test.filename, test.content, test.candidates)
assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected))
}
}
func (s *enryTestSuite) TestGetLanguageByContent() {
tests := []struct {
name string
filename string
content []byte
expected string
}{
{name: "TestGetLanguageByContent_0", filename: "", expected: ""},
{name: "TestGetLanguageByContent_1", filename: "foo.cpp", content: []byte("int main() { return 0; }"), expected: ""}, // as .cpp is unambiguous ¯\_(ツ)_/¯
{name: "TestGetLanguageByContent_2", filename: "foo.h", content: []byte("int main() { return 0; }"), expected: "C"}, // C, as it does not match any of the heuristics for C++ or Objective-C
{name: "TestGetLanguageByContent_3", filename: "foo.h", content: []byte("#include <string>\n int main() { return 0; }"), expected: "C++"}, // '#include <string>' matches regex heuristic
}
for _, test := range tests {
languages, _ := GetLanguageByContent(test.filename, test.content)
assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected))
}
}
func (s *enryTestSuite) TestGetLanguagesByExtension() {
tests := []struct {
name string
filename string
content []byte
candidates []string
expected []string
}{
{name: "TestGetLanguagesByExtension_0", filename: "foo.h", expected: []string{"C", "C++", "Objective-C"}},
{name: "TestGetLanguagesByExtension_1", filename: "foo.foo", expected: nil},
{name: "TestGetLanguagesByExtension_2", filename: "foo.go", expected: []string{"Go"}},
{name: "TestGetLanguagesByExtension_3", filename: "foo.go.php", expected: []string{"Hack", "PHP"}},
{name: "TestGetLanguagesByExtension_4", filename: "", expected: nil},
}
for _, test := range tests {
languages := GetLanguagesByExtension(test.filename, test.content, test.candidates)
assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected))
}
}
func (s *enryTestSuite) TestGetLanguagesByManpage() {
tests := []struct {
name string
filename string
content []byte
candidates []string
expected []string
}{
{name: "TestGetLanguagesByManpage_1", filename: "bsdmalloc.3malloc", expected: []string{"Roff Manpage", "Roff"}},
{name: "TestGetLanguagesByManpage_2", filename: "dirent.h.0p", expected: []string{"Roff Manpage", "Roff"}},
{name: "TestGetLanguagesByManpage_3", filename: "linguist.1gh", expected: []string{"Roff Manpage", "Roff"}},
{name: "TestGetLanguagesByManpage_4", filename: "test.1.in", expected: []string{"Roff Manpage", "Roff"}},
{name: "TestGetLanguagesByManpage_5", filename: "test.man.in", expected: []string{"Roff Manpage", "Roff"}},
{name: "TestGetLanguagesByManpage_6", filename: "test.mdoc.in", expected: []string{"Roff Manpage", "Roff"}},
{name: "TestGetLanguagesByManpage_7", filename: "foo.h", expected: nil},
{name: "TestGetLanguagesByManpage_8", filename: "", expected: nil},
}
for _, test := range tests {
languages := GetLanguagesByManpage(test.filename, test.content, test.candidates)
assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected))
}
}
func (s *enryTestSuite) TestGetLanguagesByXML() {
tests := []struct {
name string
filename string
candidates []string
expected []string
}{
{name: "TestGetLanguagesByXML_1", filename: filepath.Join(s.testFixturesDir, "XML/app.config"), expected: []string{"XML"}},
{name: "TestGetLanguagesByXML_2", filename: filepath.Join(s.testFixturesDir, "XML/AssertionIDRequestOptionalAttributes.xml.svn-base"), expected: []string{"XML"}},
// no XML header so should not be identified by this strategy
{name: "TestGetLanguagesByXML_3", filename: filepath.Join(s.samplesDir, "XML/libsomething.dll.config"), expected: nil},
{name: "TestGetLanguagesByXML_4", filename: filepath.Join(s.samplesDir, "Eagle/Eagle.sch"), candidates: []string{"Eagle"}, expected: []string{"Eagle"}},
}
for _, test := range tests {
content, err := ioutil.ReadFile(test.filename)
assert.NoError(s.T(), err)
languages := GetLanguagesByXML(test.filename, content, test.candidates)
assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected))
}
}
func (s *enryTestSuite) TestGetLanguagesByClassifier() {
test := []struct {
name string
filename string
candidates []string
expected string
}{
{name: "TestGetLanguagesByClassifier_1", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"python", "ruby", "c", "c++"}, expected: "C"},
{name: "TestGetLanguagesByClassifier_2", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: nil, expected: OtherLanguage},
{name: "TestGetLanguagesByClassifier_3", filename: filepath.Join(s.samplesDir, "C++/runtime-compiler.cc"), candidates: []string{}, expected: OtherLanguage},
{name: "TestGetLanguagesByClassifier_4", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"python", "ruby", "c++"}, expected: "C++"},
{name: "TestGetLanguagesByClassifier_5", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"ruby"}, expected: "Ruby"},
{name: "TestGetLanguagesByClassifier_6", filename: filepath.Join(s.samplesDir, "Python/django-models-base.py"), candidates: []string{"python", "ruby", "c", "c++"}, expected: "Python"},
{name: "TestGetLanguagesByClassifier_7", filename: "", candidates: []string{"python"}, expected: "Python"},
}
for _, test := range test {
var content []byte
var err error
if test.filename != "" {
content, err = ioutil.ReadFile(test.filename)
assert.NoError(s.T(), err)
}
languages := GetLanguagesByClassifier(test.filename, content, test.candidates)
var language string
if len(languages) == 0 {
language = OtherLanguage
} else {
language = languages[0]
}
assert.Equal(s.T(), test.expected, language, fmt.Sprintf("%v: language = %v, expected: %v", test.name, language, test.expected))
}
}
func (s *enryTestSuite) TestGetLanguagesBySpecificClassifier() {
test := []struct {
name string
filename string
candidates []string
classifier classifier
expected string
}{
{name: "TestGetLanguagesByClassifier_1", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"python", "ruby", "c", "c++"}, classifier: defaultClassifier, expected: "C"},
{name: "TestGetLanguagesByClassifier_2", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: nil, classifier: defaultClassifier, expected: "C"},
{name: "TestGetLanguagesByClassifier_3", filename: filepath.Join(s.samplesDir, "C++/runtime-compiler.cc"), candidates: []string{}, classifier: defaultClassifier, expected: "C++"},
{name: "TestGetLanguagesByClassifier_4", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"python", "ruby", "c++"}, classifier: defaultClassifier, expected: "C++"},
{name: "TestGetLanguagesByClassifier_5", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"ruby"}, classifier: defaultClassifier, expected: "Ruby"},
{name: "TestGetLanguagesByClassifier_6", filename: filepath.Join(s.samplesDir, "Python/django-models-base.py"), candidates: []string{"python", "ruby", "c", "c++"}, classifier: defaultClassifier, expected: "Python"},
{name: "TestGetLanguagesByClassifier_7", filename: os.DevNull, candidates: nil, classifier: defaultClassifier, expected: "XML"},
}
for _, test := range test {
content, err := ioutil.ReadFile(test.filename)
assert.NoError(s.T(), err)
languages := getLanguagesBySpecificClassifier(content, test.candidates, test.classifier)
var language string
if len(languages) == 0 {
language = OtherLanguage
} else {
language = languages[0]
}
assert.Equal(s.T(), test.expected, language, fmt.Sprintf("%v: language = %v, expected: %v", test.name, language, test.expected))
}
}
func (s *enryTestSuite) TestGetLanguageExtensions() {
tests := []struct {
name string
language string
expected []string
}{
{name: "TestGetLanguageExtensions_1", language: "foo", expected: nil},
{name: "TestGetLanguageExtensions_2", language: "COBOL", expected: []string{".cob", ".cbl", ".ccp", ".cobol", ".cpy"}},
{name: "TestGetLanguageExtensions_3", language: "Maven POM", expected: nil},
}
for _, test := range tests {
extensions := GetLanguageExtensions(test.language)
assert.EqualValues(s.T(), test.expected, extensions, fmt.Sprintf("%v: extensions = %v, expected: %v", test.name, extensions, test.expected))
}
}
func (s *enryTestSuite) TestGetLanguageType() {
tests := []struct {
name string
language string
expected Type
}{
{name: "TestGetLanguageType_1", language: "BestLanguageEver", expected: Unknown},
{name: "TestGetLanguageType_2", language: "JSON", expected: Data},
{name: "TestGetLanguageType_3", language: "COLLADA", expected: Data},
{name: "TestGetLanguageType_4", language: "Go", expected: Programming},
{name: "TestGetLanguageType_5", language: "Brainfuck", expected: Programming},
{name: "TestGetLanguageType_6", language: "HTML", expected: Markup},
{name: "TestGetLanguageType_7", language: "Sass", expected: Markup},
{name: "TestGetLanguageType_8", language: "AsciiDoc", expected: Prose},
{name: "TestGetLanguageType_9", language: "Textile", expected: Prose},
}
for _, test := range tests {
langType := GetLanguageType(test.language)
assert.Equal(s.T(), test.expected, langType, fmt.Sprintf("%v: langType = %v, expected: %v", test.name, langType, test.expected))
}
}
func (s *enryTestSuite) TestGetLanguageGroup() {
tests := []struct {
name string
language string
expected string
}{
{name: "TestGetLanguageGroup_1", language: "BestLanguageEver", expected: ""},
{name: "TestGetLanguageGroup_2", language: "Bison", expected: "Yacc"},
{name: "TestGetLanguageGroup_3", language: "HTML+PHP", expected: "HTML"},
{name: "TestGetLanguageGroup_4", language: "HTML", expected: ""},
}
for _, test := range tests {
langGroup := GetLanguageGroup(test.language)
assert.Equal(s.T(), test.expected, langGroup, fmt.Sprintf("%v: langGroup = %v, expected: %v", test.name, langGroup, test.expected))
}
}
func (s *enryTestSuite) TestGetLanguageByAlias() {
tests := []struct {
name string
alias string
expectedLang string
expectedOk bool
}{
{name: "TestGetLanguageByAlias_1", alias: "BestLanguageEver", expectedLang: OtherLanguage, expectedOk: false},
{name: "TestGetLanguageByAlias_2", alias: "aspx-vb", expectedLang: "ASP.NET", expectedOk: true},
{name: "TestGetLanguageByAlias_3", alias: "C++", expectedLang: "C++", expectedOk: true},
{name: "TestGetLanguageByAlias_4", alias: "c++", expectedLang: "C++", expectedOk: true},
{name: "TestGetLanguageByAlias_5", alias: "objc", expectedLang: "Objective-C", expectedOk: true},
{name: "TestGetLanguageByAlias_6", alias: "golang", expectedLang: "Go", expectedOk: true},
{name: "TestGetLanguageByAlias_7", alias: "GOLANG", expectedLang: "Go", expectedOk: true},
{name: "TestGetLanguageByAlias_8", alias: "bsdmake", expectedLang: "Makefile", expectedOk: true},
{name: "TestGetLanguageByAlias_9", alias: "xhTmL", expectedLang: "HTML", expectedOk: true},
{name: "TestGetLanguageByAlias_10", alias: "python", expectedLang: "Python", expectedOk: true},
}
for _, test := range tests {
lang, ok := GetLanguageByAlias(test.alias)
assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang))
assert.Equal(s.T(), test.expectedOk, ok, fmt.Sprintf("%v: ok = %v, expected: %v", test.name, ok, test.expectedOk))
}
}
func (s *enryTestSuite) TestGetLanguageID() {
tests := []struct {
name string
language string
expectedID int
found bool
}{
{name: "TestGetLanguageID_1", language: "1C Enterprise", expectedID: 0, found: true},
{name: "TestGetLanguageID_2", language: "BestLanguageEver", expectedID: 0, found: false},
{name: "TestGetLanguageID_3", language: "C++", expectedID: 43, found: true},
{name: "TestGetLanguageID_5", language: "Objective-C", expectedID: 257, found: true},
{name: "TestGetLanguageID_6", language: "golang", expectedID: 0, found: false}, // Aliases are not supported
{name: "TestGetLanguageID_7", language: "Go", expectedID: 132, found: true},
{name: "TestGetLanguageID_8", language: "Makefile", expectedID: 220, found: true},
}
for _, test := range tests {
id, found := GetLanguageID(test.language)
assert.Equal(s.T(), test.expectedID, id, fmt.Sprintf("%v: id = %v, expected: %v", test.name, id, test.expectedID))
assert.Equal(s.T(), test.found, found, fmt.Sprintf("%v: found = %t, expected: %t", test.name, found, test.found))
}
}
func (s *enryTestSuite) TestGetLanguageInfo() {
tests := []struct {
name string
language string
expectedID int
error bool
}{
{name: "TestGetLanguageID_1", language: "1C Enterprise", expectedID: 0},
{name: "TestGetLanguageID_2", language: "BestLanguageEver", error: true},
{name: "TestGetLanguageID_3", language: "C++", expectedID: 43},
{name: "TestGetLanguageID_5", language: "Objective-C", expectedID: 257},
{name: "TestGetLanguageID_6", language: "golang", error: true}, // Aliases are not supported
{name: "TestGetLanguageID_7", language: "Go", expectedID: 132},
{name: "TestGetLanguageID_8", language: "Makefile", expectedID: 220},
}
for _, test := range tests {
info, err := GetLanguageInfo(test.language)
if test.error {
assert.Error(s.T(), err, "%v: expected error for %q", test.name, test.language)
} else {
assert.NoError(s.T(), err)
assert.Equal(s.T(), test.expectedID, info.LanguageID, fmt.Sprintf("%v: id = %v, expected: %v", test.name, info.LanguageID, test.expectedID))
}
}
}
func (s *enryTestSuite) TestGetLanguageInfoByID() {
tests := []struct {
name string
id int
expectedName string
error bool
}{
{name: "TestGetLanguageID_1", id: 0, expectedName: "1C Enterprise"},
{name: "TestGetLanguageID_2", id: -1, error: true},
{name: "TestGetLanguageID_3", id: 43, expectedName: "C++"},
{name: "TestGetLanguageID_5", id: 257, expectedName: "Objective-C"},
{name: "TestGetLanguageID_7", id: 132, expectedName: "Go"},
{name: "TestGetLanguageID_8", id: 220, expectedName: "Makefile"},
}
for _, test := range tests {
info, err := GetLanguageInfoByID(test.id)
if test.error {
assert.Error(s.T(), err, "%v: expected error for %q", test.name, test.id)
} else {
assert.NoError(s.T(), err)
assert.Equal(s.T(), test.expectedName, info.Name, fmt.Sprintf("%v: id = %v, expected: %v", test.name, test.id, test.expectedName))
}
}
}

1134
data/alias.go Normal file
View File

File diff suppressed because it is too large Load Diff

601
data/colors.go Normal file
View File

@ -0,0 +1,601 @@
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 42fd3c2514375fc6ad281552368edd3fb9f6ee7f
package data
var LanguagesColor = map[string]string{
"1C Enterprise": "#814CCC",
"2-Dimensional Array": "#38761D",
"4D": "#004289",
"ABAP": "#E8274B",
"ABAP CDS": "#555e25",
"AGS Script": "#B9D9FF",
"AIDL": "#34EB6B",
"AL": "#3AA2B5",
"AMPL": "#E6EFBB",
"ANTLR": "#9DC3FF",
"API Blueprint": "#2ACCA8",
"APL": "#5A8164",
"ASP.NET": "#9400ff",
"ATS": "#1ac620",
"ActionScript": "#882B0F",
"Ada": "#02f88c",
"Adblock Filter List": "#800000",
"Adobe Font Metrics": "#fa0f00",
"Agda": "#315665",
"Alloy": "#64C800",
"Alpine Abuild": "#0D597F",
"Altium Designer": "#A89663",
"AngelScript": "#C7D7DC",
"Ant Build System": "#A9157E",
"Antlers": "#ff269e",
"ApacheConf": "#d12127",
"Apex": "#1797c0",
"Apollo Guidance Computer": "#0B3D91",
"AppleScript": "#101F1F",
"Arc": "#aa2afe",
"AsciiDoc": "#73a0c5",
"AspectJ": "#a957b0",
"Assembly": "#6E4C13",
"Astro": "#ff5a03",
"Asymptote": "#ff0000",
"Augeas": "#9CC134",
"AutoHotkey": "#6594b9",
"AutoIt": "#1C3552",
"Avro IDL": "#0040FF",
"Awk": "#c30e9b",
"BASIC": "#ff0000",
"BQN": "#2b7067",
"Ballerina": "#FF5000",
"Batchfile": "#C1F12E",
"Beef": "#a52f4e",
"Berry": "#15A13C",
"BibTeX": "#778899",
"Bicep": "#519aba",
"Bikeshed": "#5562ac",
"Bison": "#6A463F",
"BitBake": "#00bce4",
"Blade": "#f7523f",
"BlitzBasic": "#00FFAE",
"BlitzMax": "#cd6400",
"Bluespec": "#12223c",
"Bluespec BH": "#12223c",
"Boo": "#d4bec1",
"Boogie": "#c80fa0",
"Brainfuck": "#2F2530",
"BrighterScript": "#66AABB",
"Brightscript": "#662D91",
"Browserslist": "#ffd539",
"C": "#555555",
"C#": "#178600",
"C++": "#f34b7d",
"CAP CDS": "#0092d1",
"CLIPS": "#00A300",
"CMake": "#DA3434",
"COLLADA": "#F1A42B",
"CSON": "#244776",
"CSS": "#563d7c",
"CSV": "#237346",
"CUE": "#5886E1",
"CWeb": "#00007a",
"Cabal Config": "#483465",
"Caddyfile": "#22b638",
"Cadence": "#00ef8b",
"Cairo": "#ff4a48",
"CameLIGO": "#3be133",
"Cap'n Proto": "#c42727",
"Ceylon": "#dfa535",
"Chapel": "#8dc63f",
"ChucK": "#3f8000",
"Circom": "#707575",
"Cirru": "#ccccff",
"Clarion": "#db901e",
"Clarity": "#5546ff",
"Classic ASP": "#6a40fd",
"Clean": "#3F85AF",
"Click": "#E4E6F3",
"Clojure": "#db5855",
"Closure Templates": "#0d948f",
"Cloud Firestore Security Rules": "#FFA000",
"CodeQL": "#140f46",
"CoffeeScript": "#244776",
"ColdFusion": "#ed2cd6",
"ColdFusion CFC": "#ed2cd6",
"Common Lisp": "#3fb68b",
"Common Workflow Language": "#B5314C",
"Component Pascal": "#B0CE4E",
"Coq": "#d0b68c",
"Crystal": "#000100",
"Csound": "#1a1a1a",
"Csound Document": "#1a1a1a",
"Csound Score": "#1a1a1a",
"Cuda": "#3A4E3A",
"Curry": "#531242",
"Cypher": "#34c0eb",
"Cython": "#fedf5b",
"D": "#ba595e",
"D2": "#526ee8",
"DM": "#447265",
"Dafny": "#FFEC25",
"Darcs Patch": "#8eff23",
"Dart": "#00B4AB",
"DataWeave": "#003a52",
"Debian Package Control File": "#D70751",
"DenizenScript": "#FBEE96",
"Dhall": "#dfafff",
"DirectX 3D File": "#aace60",
"Dockerfile": "#384d54",
"Dogescript": "#cca760",
"Dotenv": "#e5d559",
"Dylan": "#6c616e",
"E": "#ccce35",
"ECL": "#8a1267",
"ECLiPSe": "#001d9d",
"EJS": "#a91e50",
"EQ": "#a78649",
"Earthly": "#2af0ff",
"Easybuild": "#069406",
"Ecere Projects": "#913960",
"Ecmarkup": "#eb8131",
"Edge": "#0dffe0",
"EdgeQL": "#31A7FF",
"EditorConfig": "#fff1f2",
"Eiffel": "#4d6977",
"Elixir": "#6e4a7e",
"Elm": "#60B5CC",
"Elvish": "#55BB55",
"Elvish Transcript": "#55BB55",
"Emacs Lisp": "#c065db",
"EmberScript": "#FFF4F3",
"Erlang": "#B83998",
"Euphoria": "#FF790B",
"F#": "#b845fc",
"F*": "#572e30",
"FIGlet Font": "#FFDDBB",
"FIRRTL": "#2f632f",
"FLUX": "#88ccff",
"Factor": "#636746",
"Fancy": "#7b9db4",
"Fantom": "#14253c",
"Faust": "#c37240",
"Fennel": "#fff3d7",
"Filebench WML": "#F6B900",
"Fluent": "#ffcc33",
"Forth": "#341708",
"Fortran": "#4d41b1",
"Fortran Free Form": "#4d41b1",
"FreeBasic": "#141AC9",
"FreeMarker": "#0050b2",
"Frege": "#00cafe",
"Futhark": "#5f021f",
"G-code": "#D08CF2",
"GAML": "#FFC766",
"GAMS": "#f49a22",
"GAP": "#0000cc",
"GCC Machine Description": "#FFCFAB",
"GDScript": "#355570",
"GEDCOM": "#003058",
"GLSL": "#5686a5",
"GSC": "#FF6800",
"Game Maker Language": "#71b417",
"Gemfile.lock": "#701516",
"Gemini": "#ff6900",
"Genero 4gl": "#63408e",
"Genero per": "#d8df39",
"Genie": "#fb855d",
"Genshi": "#951531",
"Gentoo Ebuild": "#9400ff",
"Gentoo Eclass": "#9400ff",
"Gerber Image": "#d20b00",
"Gherkin": "#5B2063",
"Git Attributes": "#F44D27",
"Git Config": "#F44D27",
"Git Revision List": "#F44D27",
"Gleam": "#ffaff3",
"Glimmer JS": "#F5835F",
"Glimmer TS": "#3178c6",
"Glyph": "#c1ac7f",
"Gnuplot": "#f0a9f0",
"Go": "#00ADD8",
"Go Checksums": "#00ADD8",
"Go Module": "#00ADD8",
"Go Workspace": "#00ADD8",
"Godot Resource": "#355570",
"Golo": "#88562A",
"Gosu": "#82937f",
"Grace": "#615f8b",
"Gradle": "#02303a",
"Gradle Kotlin DSL": "#02303a",
"Grammatical Framework": "#ff0000",
"GraphQL": "#e10098",
"Graphviz (DOT)": "#2596be",
"Groovy": "#4298b8",
"Groovy Server Pages": "#4298b8",
"HAProxy": "#106da9",
"HCL": "#844FBA",
"HLSL": "#aace60",
"HOCON": "#9ff8ee",
"HTML": "#e34c26",
"HTML+ECR": "#2e1052",
"HTML+EEX": "#6e4a7e",
"HTML+ERB": "#701516",
"HTML+PHP": "#4f5d95",
"HTML+Razor": "#512be4",
"HTTP": "#005C9C",
"HXML": "#f68712",
"Hack": "#878787",
"Haml": "#ece2a9",
"Handlebars": "#f7931e",
"Harbour": "#0e60e3",
"Haskell": "#5e5086",
"Haxe": "#df7900",
"HiveQL": "#dce200",
"HolyC": "#ffefaf",
"Hosts File": "#308888",
"Hy": "#7790B2",
"IDL": "#a3522f",
"IGOR Pro": "#0000cc",
"INI": "#d1dbe0",
"Idris": "#b30000",
"Ignore List": "#000000",
"ImageJ Macro": "#99AAFF",
"Imba": "#16cec6",
"Inno Setup": "#264b99",
"Io": "#a9188d",
"Ioke": "#078193",
"Isabelle": "#FEFE00",
"Isabelle ROOT": "#FEFE00",
"J": "#9EEDFF",
"JAR Manifest": "#b07219",
"JCL": "#d90e09",
"JFlex": "#DBCA00",
"JSON": "#292929",
"JSON with Comments": "#292929",
"JSON5": "#267CB9",
"JSONLD": "#0c479c",
"JSONiq": "#40d47e",
"Janet": "#0886a5",
"Jasmin": "#d03600",
"Java": "#b07219",
"Java Properties": "#2A6277",
"Java Server Pages": "#2A6277",
"JavaScript": "#f1e05a",
"JavaScript+ERB": "#f1e05a",
"Jest Snapshot": "#15c213",
"JetBrains MPS": "#21D789",
"Jinja": "#a52a22",
"Jison": "#56b3cb",
"Jison Lex": "#56b3cb",
"Jolie": "#843179",
"Jsonnet": "#0064bd",
"Julia": "#a270ba",
"Julia REPL": "#a270ba",
"Jupyter Notebook": "#DA5B0B",
"Just": "#384d54",
"KRL": "#28430A",
"Kaitai Struct": "#773b37",
"KakouneScript": "#6f8042",
"KerboScript": "#41adf0",
"KiCad Layout": "#2f4aab",
"KiCad Legacy Layout": "#2f4aab",
"KiCad Schematic": "#2f4aab",
"Kotlin": "#A97BFF",
"LFE": "#4C3023",
"LLVM": "#185619",
"LOLCODE": "#cc9900",
"LSL": "#3d9970",
"LabVIEW": "#fede06",
"Lark": "#2980B9",
"Lasso": "#999999",
"Latte": "#f2a542",
"Less": "#1d365d",
"Lex": "#DBCA00",
"LigoLANG": "#0e74ff",
"LilyPond": "#9ccc7c",
"Liquid": "#67b8de",
"Literate Agda": "#315665",
"Literate CoffeeScript": "#244776",
"Literate Haskell": "#5e5086",
"LiveScript": "#499886",
"Logtalk": "#295b9a",
"LookML": "#652B81",
"Lua": "#000080",
"Luau": "#00A2FF",
"MATLAB": "#e16737",
"MAXScript": "#00a6a6",
"MDX": "#fcb32c",
"MLIR": "#5EC8DB",
"MQL4": "#62A8D6",
"MQL5": "#4A76B8",
"MTML": "#b7e1f4",
"Macaulay2": "#d8ffff",
"Makefile": "#427819",
"Mako": "#7e858d",
"Markdown": "#083fa1",
"Marko": "#42bff2",
"Mask": "#f97732",
"Mathematica": "#dd1100",
"Max": "#c4a79c",
"Mercury": "#ff2b2b",
"Mermaid": "#ff3670",
"Meson": "#007800",
"Metal": "#8f14e9",
"MiniYAML": "#ff1111",
"Mint": "#02b046",
"Mirah": "#c7a938",
"Modelica": "#de1d31",
"Modula-2": "#10253f",
"Modula-3": "#223388",
"Mojo": "#ff4c1f",
"Monkey C": "#8D6747",
"MoonScript": "#ff4585",
"Motoko": "#fbb03b",
"Motorola 68K Assembly": "#005daa",
"Move": "#4a137a",
"Mustache": "#724b3b",
"NCL": "#28431f",
"NMODL": "#00356B",
"NPM Config": "#cb3837",
"NWScript": "#111522",
"Nasal": "#1d2c4e",
"Nearley": "#990000",
"Nemerle": "#3d3c6e",
"NetLinx": "#0aa0ff",
"NetLinx+ERB": "#747faa",
"NetLogo": "#ff6375",
"NewLisp": "#87AED7",
"Nextflow": "#3ac486",
"Nginx": "#009639",
"Nim": "#ffc200",
"Nit": "#009917",
"Nix": "#7e7eff",
"Nu": "#c9df40",
"NumPy": "#9C8AF9",
"Nunjucks": "#3d8137",
"Nushell": "#4E9906",
"OASv2-json": "#85ea2d",
"OASv2-yaml": "#85ea2d",
"OASv3-json": "#85ea2d",
"OASv3-yaml": "#85ea2d",
"OCaml": "#ef7a08",
"ObjectScript": "#424893",
"Objective-C": "#438eff",
"Objective-C++": "#6866fb",
"Objective-J": "#ff0c5a",
"Odin": "#60AFFE",
"Omgrofl": "#cabbff",
"Opal": "#f7ede0",
"Open Policy Agent": "#7d9199",
"OpenAPI Specification v2": "#85ea2d",
"OpenAPI Specification v3": "#85ea2d",
"OpenCL": "#ed2e2d",
"OpenEdge ABL": "#5ce600",
"OpenQASM": "#AA70FF",
"OpenSCAD": "#e5cd45",
"Option List": "#476732",
"Org": "#77aa99",
"Oxygene": "#cdd0e3",
"Oz": "#fab738",
"P4": "#7055b5",
"PDDL": "#0d00ff",
"PEG.js": "#234d6b",
"PHP": "#4F5D95",
"PLSQL": "#dad8d8",
"PLpgSQL": "#336790",
"POV-Ray SDL": "#6bac65",
"Pact": "#F7A8B8",
"Pan": "#cc0000",
"Papyrus": "#6600cc",
"Parrot": "#f3ca0a",
"Pascal": "#E3F171",
"Pawn": "#dbb284",
"Pep8": "#C76F5B",
"Perl": "#0298c3",
"PicoLisp": "#6067af",
"PigLatin": "#fcd7de",
"Pike": "#005390",
"Pip Requirements": "#FFD343",
"Pkl": "#6b9543",
"PlantUML": "#fbbd16",
"PogoScript": "#d80074",
"Polar": "#ae81ff",
"Portugol": "#f8bd00",
"PostCSS": "#dc3a0c",
"PostScript": "#da291c",
"PowerBuilder": "#8f0f8d",
"PowerShell": "#012456",
"Praat": "#c8506d",
"Prisma": "#0c344b",
"Processing": "#0096D8",
"Procfile": "#3B2F63",
"Prolog": "#74283c",
"Promela": "#de0000",
"Propeller Spin": "#7fa2a7",
"Pug": "#a86454",
"Puppet": "#302B6D",
"PureBasic": "#5a6986",
"PureScript": "#1D222D",
"Pyret": "#ee1e10",
"Python": "#3572A5",
"Python console": "#3572A5",
"Python traceback": "#3572A5",
"Q#": "#fed659",
"QML": "#44a51c",
"Qt Script": "#00b841",
"Quake": "#882233",
"R": "#198CE7",
"RAML": "#77d9fb",
"RBS": "#701516",
"RDoc": "#701516",
"REXX": "#d90e09",
"RMarkdown": "#198ce7",
"RON": "#a62c00",
"RPGLE": "#2BDE21",
"RUNOFF": "#665a4e",
"Racket": "#3c5caa",
"Ragel": "#9d5200",
"Raku": "#0000fb",
"Rascal": "#fffaa0",
"ReScript": "#ed5051",
"Reason": "#ff5847",
"ReasonLIGO": "#ff5847",
"Rebol": "#358a5b",
"Record Jar": "#0673ba",
"Red": "#f50000",
"Regular Expression": "#009a00",
"Ren'Py": "#ff7f7f",
"Rez": "#FFDAB3",
"Ring": "#2D54CB",
"Riot": "#A71E49",
"RobotFramework": "#00c0b5",
"Roc": "#7c38f5",
"Roff": "#ecdebe",
"Roff Manpage": "#ecdebe",
"Rouge": "#cc0088",
"RouterOS Script": "#DE3941",
"Ruby": "#701516",
"Rust": "#dea584",
"SAS": "#B34936",
"SCSS": "#c6538c",
"SPARQL": "#0C4597",
"SQF": "#3F3F3F",
"SQL": "#e38c00",
"SQLPL": "#e38c00",
"SRecode Template": "#348a34",
"STL": "#373b5e",
"SVG": "#ff9900",
"SaltStack": "#646464",
"Sass": "#a53b70",
"Scala": "#c22d40",
"Scaml": "#bd181a",
"Scenic": "#fdc700",
"Scheme": "#1e4aec",
"Scilab": "#ca0f21",
"Self": "#0579aa",
"ShaderLab": "#222c37",
"Shell": "#89e051",
"ShellCheck Config": "#cecfcb",
"Shen": "#120F14",
"Simple File Verification": "#C9BFED",
"Singularity": "#64E6AD",
"Slash": "#007eff",
"Slice": "#003fa2",
"Slim": "#2b2b2b",
"Slint": "#2379F4",
"SmPL": "#c94949",
"Smalltalk": "#596706",
"Smarty": "#f0c040",
"Smithy": "#c44536",
"Snakemake": "#419179",
"Solidity": "#AA6746",
"SourcePawn": "#f69e1d",
"Squirrel": "#800000",
"Stan": "#b2011d",
"Standard ML": "#dc566d",
"Starlark": "#76d275",
"Stata": "#1a5f91",
"StringTemplate": "#3fb34f",
"Stylus": "#ff6347",
"SubRip Text": "#9e0101",
"SugarSS": "#2fcc9f",
"SuperCollider": "#46390b",
"Svelte": "#ff3e00",
"Sway": "#00F58C",
"Sweave": "#198ce7",
"Swift": "#F05138",
"SystemVerilog": "#DAE1C2",
"TI Program": "#A0AA87",
"TL-Verilog": "#C40023",
"TLA": "#4b0079",
"TOML": "#9c4221",
"TSQL": "#e38c00",
"TSV": "#237346",
"TSX": "#3178c6",
"TXL": "#0178b8",
"Talon": "#333333",
"Tcl": "#e4cc98",
"TeX": "#3D6117",
"Terra": "#00004c",
"Terraform Template": "#7b42bb",
"TextGrid": "#c8506d",
"TextMate Properties": "#df66e4",
"Textile": "#ffe7ac",
"Thrift": "#D12127",
"Toit": "#c2c9fb",
"Turing": "#cf142b",
"Twig": "#c1d026",
"TypeScript": "#3178c6",
"Typst": "#239dad",
"Unified Parallel C": "#4e3617",
"Unity3D Asset": "#222c37",
"Uno": "#9933cc",
"UnrealScript": "#a54c4d",
"UrWeb": "#ccccee",
"V": "#4f87c4",
"VBA": "#867db1",
"VBScript": "#15dcdc",
"VCL": "#148AA8",
"VHDL": "#adb2cb",
"Vala": "#a56de2",
"Valve Data Format": "#f26025",
"Velocity Template Language": "#507cff",
"Verilog": "#b2b7f8",
"Vim Help File": "#199f4b",
"Vim Script": "#199f4b",
"Vim Snippet": "#199f4b",
"Visual Basic .NET": "#945db7",
"Visual Basic 6.0": "#2c6353",
"Volt": "#1F1F1F",
"Vue": "#41b883",
"Vyper": "#2980b9",
"WDL": "#42f1f4",
"WGSL": "#1a5e9a",
"Web Ontology Language": "#5b70bd",
"WebAssembly": "#04133b",
"WebAssembly Interface Type": "#6250e7",
"Whiley": "#d5c397",
"Wikitext": "#fc5757",
"Windows Registry Entries": "#52d5ff",
"Witcher Script": "#ff0000",
"Wollok": "#a23738",
"World of Warcraft Addon Data": "#f7e43f",
"Wren": "#383838",
"X10": "#4B6BEF",
"XC": "#99DA07",
"XML": "#0060ac",
"XML Property List": "#0060ac",
"XQuery": "#5232e7",
"XSLT": "#EB8CEB",
"Xojo": "#81bd41",
"Xonsh": "#285EEF",
"Xtend": "#24255d",
"YAML": "#cb171e",
"YARA": "#220000",
"YASnippet": "#32AB90",
"Yacc": "#4B6C4B",
"Yul": "#794932",
"ZAP": "#0d665e",
"ZIL": "#dc75e5",
"ZenScript": "#00BCD1",
"Zephir": "#118f9e",
"Zig": "#ec915c",
"Zimpl": "#d67711",
"crontab": "#ead7ac",
"eC": "#913960",
"fish": "#4aae47",
"hoon": "#00b171",
"jq": "#c7254e",
"kvlang": "#1da6e0",
"mIRC Script": "#3d57c3",
"mcfunction": "#E22837",
"mupad": "#244963",
"nanorc": "#2d004d",
"nesC": "#94B0C7",
"ooc": "#b0b77e",
"q": "#0040cd",
"reStructuredText": "#141414",
"sed": "#64b970",
"templ": "#66D0DD",
"wisp": "#7582D1",
"xBase": "#403a40",
}

7
data/commit.go Normal file
View File

@ -0,0 +1,7 @@
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 42fd3c2514375fc6ad281552368edd3fb9f6ee7f
package data
// linguist's commit from which files were generated.
var LinguistCommit = "42fd3c2514375fc6ad281552368edd3fb9f6ee7f"

1972
data/content.go Normal file
View File

File diff suppressed because it is too large Load Diff

3
data/doc.go Normal file
View File

@ -0,0 +1,3 @@
// Package data contains only auto-generated data-structures for all the language
// identification strategies from the Linguist project sources.
package data

27
data/documentation.go Normal file
View File

@ -0,0 +1,27 @@
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 42fd3c2514375fc6ad281552368edd3fb9f6ee7f
package data
import "github.com/go-enry/go-enry/v2/regex"
var DocumentationMatchers = []regex.EnryRegexp{
regex.MustCompile(`^[Dd]ocs?/`),
regex.MustCompile(`(^|/)[Dd]ocumentation/`),
regex.MustCompile(`(^|/)[Gg]roovydoc/`),
regex.MustCompile(`(^|/)[Jj]avadoc/`),
regex.MustCompile(`^[Mm]an/`),
regex.MustCompile(`^[Ee]xamples/`),
regex.MustCompile(`^[Dd]emos?/`),
regex.MustCompile(`(^|/)inst/doc/`),
regex.MustCompile(`(^|/)CITATION(\.cff|(S)?(\.(bib|md))?)$`),
regex.MustCompile(`(^|/)CHANGE(S|LOG)?(\.|$)`),
regex.MustCompile(`(^|/)CONTRIBUTING(\.|$)`),
regex.MustCompile(`(^|/)COPYING(\.|$)`),
regex.MustCompile(`(^|/)INSTALL(\.|$)`),
regex.MustCompile(`(^|/)LICEN[CS]E(\.|$)`),
regex.MustCompile(`(^|/)[Ll]icen[cs]e(\.|$)`),
regex.MustCompile(`(^|/)README(\.|$)`),
regex.MustCompile(`(^|/)[Rr]eadme(\.|$)`),
regex.MustCompile(`^[Ss]amples?/`),
}

2062
data/extension.go Normal file
View File

File diff suppressed because it is too large Load Diff

376
data/filename.go Normal file
View File

@ -0,0 +1,376 @@
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 42fd3c2514375fc6ad281552368edd3fb9f6ee7f
package data
var LanguagesByFilename = map[string][]string{
".JUSTFILE": {"Just"},
".Justfile": {"Just"},
".Rprofile": {"R"},
".XCompose": {"XCompose"},
".abbrev_defs": {"Emacs Lisp"},
".ackrc": {"Option List"},
".all-contributorsrc": {"JSON"},
".arcconfig": {"JSON"},
".atomignore": {"Ignore List"},
".auto-changelog": {"JSON"},
".babelignore": {"Ignore List"},
".babelrc": {"JSON with Comments"},
".bash_aliases": {"Shell"},
".bash_functions": {"Shell"},
".bash_history": {"Shell"},
".bash_logout": {"Shell"},
".bash_profile": {"Shell"},
".bashrc": {"Shell"},
".browserslistrc": {"Browserslist"},
".bzrignore": {"Ignore List"},
".c8rc": {"JSON"},
".clang-format": {"YAML"},
".clang-tidy": {"YAML"},
".classpath": {"XML"},
".coffeelintignore": {"Ignore List"},
".coveragerc": {"INI"},
".cproject": {"XML"},
".cshrc": {"Shell"},
".curlrc": {"cURL Config"},
".cvsignore": {"Ignore List"},
".devcontainer.json": {"JSON with Comments"},
".dir_colors": {"dircolors"},
".dircolors": {"dircolors"},
".dockerignore": {"Ignore List"},
".editorconfig": {"EditorConfig"},
".eleventyignore": {"Ignore List"},
".emacs": {"Emacs Lisp"},
".emacs.desktop": {"Emacs Lisp"},
".env": {"Dotenv"},
".env.ci": {"Dotenv"},
".env.dev": {"Dotenv"},
".env.development": {"Dotenv"},
".env.development.local": {"Dotenv"},
".env.example": {"Dotenv"},
".env.local": {"Dotenv"},
".env.prod": {"Dotenv"},
".env.production": {"Dotenv"},
".env.sample": {"Dotenv"},
".env.staging": {"Dotenv"},
".env.test": {"Dotenv"},
".env.testing": {"Dotenv"},
".envrc": {"Shell"},
".eslintignore": {"Ignore List"},
".eslintrc.json": {"JSON with Comments"},
".exrc": {"Vim Script"},
".factor-boot-rc": {"Factor"},
".factor-rc": {"Factor"},
".flake8": {"INI"},
".flaskenv": {"Shell"},
".gclient": {"Python"},
".gemrc": {"YAML"},
".git-blame-ignore-revs": {"Git Revision List"},
".gitattributes": {"Git Attributes"},
".gitconfig": {"Git Config"},
".gitignore": {"Ignore List"},
".gitmodules": {"Git Config"},
".gn": {"GN"},
".gnus": {"Emacs Lisp"},
".gvimrc": {"Vim Script"},
".htaccess": {"ApacheConf"},
".htmlhintrc": {"JSON"},
".imgbotconfig": {"JSON"},
".inputrc": {"Readline Config"},
".irbrc": {"Ruby"},
".jscsrc": {"JSON with Comments"},
".jshintrc": {"JSON with Comments"},
".jslintrc": {"JSON with Comments"},
".justfile": {"Just"},
".kshrc": {"Shell"},
".latexmkrc": {"Perl"},
".login": {"Shell"},
".luacheckrc": {"Lua"},
".markdownlintignore": {"Ignore List"},
".nanorc": {"nanorc"},
".nodemonignore": {"Ignore List"},
".npmignore": {"Ignore List"},
".npmrc": {"NPM Config"},
".nvimrc": {"Vim Script"},
".nycrc": {"JSON"},
".php": {"PHP"},
".php_cs": {"PHP"},
".php_cs.dist": {"PHP"},
".prettierignore": {"Ignore List"},
".profile": {"Shell"},
".project": {"XML"},
".pryrc": {"Ruby"},
".pylintrc": {"INI"},
".rspec": {"Option List"},
".scalafix.conf": {"HOCON"},
".scalafmt.conf": {"HOCON"},
".shellcheckrc": {"ShellCheck Config"},
".simplecov": {"Ruby"},
".spacemacs": {"Emacs Lisp"},
".stylelintignore": {"Ignore List"},
".swcrc": {"JSON with Comments"},
".tern-config": {"JSON"},
".tern-project": {"JSON"},
".tm_properties": {"TextMate Properties"},
".tmux.conf": {"Shell"},
".vercelignore": {"Ignore List"},
".vimrc": {"Vim Script"},
".viper": {"Emacs Lisp"},
".vscodeignore": {"Ignore List"},
".watchmanconfig": {"JSON"},
".wgetrc": {"Wget Config"},
".yardopts": {"Option List"},
".zlogin": {"Shell"},
".zlogout": {"Shell"},
".zprofile": {"Shell"},
".zshenv": {"Shell"},
".zshrc": {"Shell"},
"9fs": {"Shell"},
"APKBUILD": {"Alpine Abuild"},
"Android.bp": {"Soong"},
"App.config": {"XML"},
"Appraisals": {"Ruby"},
"BSDmakefile": {"Makefile"},
"BUCK": {"Starlark"},
"BUILD": {"Starlark"},
"BUILD.bazel": {"Starlark"},
"Berksfile": {"Ruby"},
"Brewfile": {"Ruby"},
"Buildfile": {"Ruby"},
"CITATION": {"Text"},
"CITATION.cff": {"YAML"},
"CITATIONS": {"Text"},
"CMakeLists.txt": {"CMake"},
"CODEOWNERS": {"CODEOWNERS"},
"COPYING": {"Text"},
"COPYING.regex": {"Text"},
"COPYRIGHT.regex": {"Text"},
"Caddyfile": {"Caddyfile"},
"Cakefile": {"CoffeeScript"},
"Capfile": {"Ruby"},
"Cargo.lock": {"TOML"},
"Cargo.toml.orig": {"TOML"},
"Cask": {"Emacs Lisp"},
"Containerfile": {"Dockerfile"},
"DEPS": {"Python"},
"DIR_COLORS": {"dircolors"},
"Dangerfile": {"Ruby"},
"Deliverfile": {"Ruby"},
"Dockerfile": {"Dockerfile"},
"Earthfile": {"Earthly"},
"Emakefile": {"Erlang"},
"FONTLOG": {"Text"},
"Fakefile": {"Fancy"},
"Fastfile": {"Ruby"},
"GNUmakefile": {"Makefile"},
"Gemfile": {"Ruby"},
"Gemfile.lock": {"Gemfile.lock"},
"Gopkg.lock": {"TOML"},
"Guardfile": {"Ruby"},
"HOSTS": {"Hosts File", "INI"},
"INSTALL": {"Text"},
"INSTALL.mysql": {"Text"},
"JUSTFILE": {"Just"},
"Jakefile": {"JavaScript"},
"Jarfile": {"Ruby"},
"Jenkinsfile": {"Groovy"},
"Justfile": {"Just"},
"Kbuild": {"Makefile"},
"LICENSE": {"Text"},
"LICENSE.mysql": {"Text"},
"Lexer.x": {"Lex"},
"MANIFEST.MF": {"JAR Manifest"},
"MD5SUMS": {"Checksums"},
"MODULE.bazel": {"Starlark"},
"MODULE.bazel.lock": {"JSON"},
"Makefile": {"Makefile"},
"Makefile.PL": {"Perl"},
"Makefile.am": {"Makefile"},
"Makefile.boot": {"Makefile"},
"Makefile.frag": {"Makefile"},
"Makefile.in": {"Makefile"},
"Makefile.inc": {"Makefile"},
"Makefile.wat": {"Makefile"},
"Mavenfile": {"Ruby"},
"Modulefile": {"Puppet"},
"NEWS": {"Text"},
"Notebook": {"Jupyter Notebook"},
"NuGet.config": {"XML"},
"Nukefile": {"Nu"},
"PKGBUILD": {"Shell"},
"Phakefile": {"PHP"},
"Pipfile": {"TOML"},
"Pipfile.lock": {"JSON"},
"Podfile": {"Ruby"},
"Procfile": {"Procfile"},
"Project.ede": {"Emacs Lisp"},
"Puppetfile": {"Ruby"},
"README.me": {"Text"},
"README.mysql": {"Text"},
"README.nss": {"Text"},
"ROOT": {"Isabelle ROOT"},
"Rakefile": {"Ruby"},
"Rexfile": {"Perl"},
"SConscript": {"Python"},
"SConstruct": {"Python"},
"SHA1SUMS": {"Checksums"},
"SHA256SUMS": {"Checksums"},
"SHA256SUMS.txt": {"Checksums"},
"SHA512SUMS": {"Checksums"},
"Settings.StyleCop": {"XML"},
"Singularity": {"Singularity"},
"Slakefile": {"LiveScript"},
"Snakefile": {"Snakemake"},
"Snapfile": {"Ruby"},
"Steepfile": {"Ruby"},
"Thorfile": {"Ruby"},
"Tiltfile": {"Starlark"},
"Vagrantfile": {"Ruby"},
"WORKSPACE": {"Starlark"},
"WORKSPACE.bazel": {"Starlark"},
"Web.Debug.config": {"XML"},
"Web.Release.config": {"XML"},
"Web.config": {"XML"},
"XCompose": {"XCompose"},
"_curlrc": {"cURL Config"},
"_dir_colors": {"dircolors"},
"_dircolors": {"dircolors"},
"_emacs": {"Emacs Lisp"},
"_redirects": {"Redirect Rules"},
"_vimrc": {"Vim Script"},
"abbrev_defs": {"Emacs Lisp"},
"ack": {"Perl"},
"ackrc": {"Option List"},
"ant.xml": {"Ant Build System"},
"apache2.conf": {"ApacheConf"},
"api-extractor.json": {"JSON with Comments"},
"bash_aliases": {"Shell"},
"bash_logout": {"Shell"},
"bash_profile": {"Shell"},
"bashrc": {"Shell"},
"browserslist": {"Browserslist"},
"build.xml": {"Ant Build System"},
"buildfile": {"Ruby"},
"buildozer.spec": {"INI"},
"cabal.config": {"Cabal Config"},
"cabal.project": {"Cabal Config"},
"checksums.txt": {"Checksums"},
"cksums": {"Checksums"},
"click.me": {"Text"},
"composer.lock": {"JSON"},
"configure.ac": {"M4Sugar"},
"contents.lr": {"Markdown"},
"cpanfile": {"Perl"},
"crontab": {"crontab"},
"cshrc": {"Shell"},
"delete.me": {"Text"},
"deno.lock": {"JSON"},
"descrip.mmk": {"Module Management System"},
"descrip.mms": {"Module Management System"},
"devcontainer.json": {"JSON with Comments"},
"dir_colors": {"dircolors"},
"encodings.dir": {"X Font Directory Index"},
"eqnrc": {"Roff"},
"expr-dist": {"R"},
"file_contexts": {"SELinux Policy"},
"firestore.rules": {"Cloud Firestore Security Rules"},
"flake.lock": {"JSON"},
"fonts.alias": {"X Font Directory Index"},
"fonts.dir": {"X Font Directory Index"},
"fonts.scale": {"X Font Directory Index"},
"fp-lib-table": {"KiCad Layout"},
"genfs_contexts": {"SELinux Policy"},
"gitignore-global": {"Ignore List"},
"gitignore_global": {"Ignore List"},
"glide.lock": {"YAML"},
"go.mod": {"Go Module"},
"go.sum": {"Go Checksums"},
"go.work": {"Go Workspace"},
"go.work.sum": {"Go Checksums"},
"gradlew": {"Shell"},
"gvimrc": {"Vim Script"},
"haproxy.cfg": {"HAProxy"},
"hosts": {"Hosts File", "INI"},
"httpd.conf": {"ApacheConf"},
"initial_sids": {"SELinux Policy"},
"inputrc": {"Readline Config"},
"installscript.qs": {"Qt Script"},
"jsconfig.json": {"JSON with Comments"},
"justfile": {"Just"},
"kakrc": {"KakouneScript"},
"keep.me": {"Text"},
"kshrc": {"Shell"},
"language-configuration.json": {"JSON with Comments"},
"language-subtag-registry.txt": {"Record Jar"},
"latexmkrc": {"Perl"},
"ld.script": {"Linker Script"},
"lexer.x": {"Lex"},
"login": {"Shell"},
"m3makefile": {"Quake"},
"m3overrides": {"Quake"},
"makefile": {"Makefile"},
"makefile.sco": {"Makefile"},
"man": {"Shell"},
"mcmod.info": {"JSON"},
"md5sum.txt": {"Checksums"},
"meson.build": {"Meson"},
"meson_options.txt": {"Meson"},
"mix.lock": {"Elixir"},
"mkfile": {"Makefile"},
"mmn": {"Roff"},
"mmt": {"Roff"},
"mocha.opts": {"Option List"},
"nanorc": {"nanorc"},
"nextflow.config": {"Nextflow"},
"nginx.conf": {"Nginx"},
"nim.cfg": {"Nim"},
"nvimrc": {"Vim Script"},
"owh": {"Tcl"},
"package.mask": {"Text"},
"package.use.mask": {"Text"},
"package.use.stable.mask": {"Text"},
"packages.config": {"XML"},
"pdm.lock": {"TOML"},
"poetry.lock": {"TOML"},
"pom.xml": {"Maven POM"},
"port_contexts": {"SELinux Policy"},
"profile": {"Shell"},
"project.godot": {"Godot Resource"},
"pylintrc": {"INI"},
"read.me": {"Text"},
"readme.1st": {"Text"},
"rebar.config": {"Erlang"},
"rebar.config.lock": {"Erlang"},
"rebar.lock": {"Erlang"},
"requirements-dev.txt": {"Pip Requirements"},
"requirements.txt": {"Pip Requirements"},
"riemann.config": {"Clojure"},
"robots.txt": {"robots.txt"},
"security_classes": {"SELinux Policy"},
"ssh-config": {"SSH Config"},
"ssh_config": {"SSH Config"},
"sshconfig": {"SSH Config"},
"sshconfig.snip": {"SSH Config"},
"sshd-config": {"SSH Config"},
"sshd_config": {"SSH Config"},
"starfield": {"Tcl"},
"test.me": {"Text"},
"tmux.conf": {"Shell"},
"toolchain_installscript.qs": {"Qt Script"},
"troffrc": {"Roff"},
"troffrc-end": {"Roff"},
"tsconfig.json": {"JSON with Comments"},
"tslint.json": {"JSON with Comments"},
"use.mask": {"Text"},
"use.stable.mask": {"Text"},
"vimrc": {"Vim Script"},
"vlcrc": {"INI"},
"wscript": {"Python"},
"xcompose": {"XCompose"},
"yarn.lock": {"YAML"},
"zlogin": {"Shell"},
"zlogout": {"Shell"},
"zprofile": {"Shell"},
"zshenv": {"Shell"},
"zshrc": {"Shell"},
}

225778
data/frequencies.go Normal file
View File

File diff suppressed because it is too large Load Diff

845
data/generated.go Normal file
View File

@ -0,0 +1,845 @@
package data
import (
"bytes"
"strings"
"github.com/go-enry/go-enry/v2/regex"
)
// GeneratedCodeExtensions contains all extensions that belong to generated
// files for sure.
var GeneratedCodeExtensions = map[string]struct{}{
// XCode files
".nib": {},
".xcworkspacedata": {},
".xcuserstate": {},
}
// GeneratedCodeNameMatcher is a function that tells whether the file with the
// given name is generated.
type GeneratedCodeNameMatcher func(string) bool
func nameMatches(pattern string) GeneratedCodeNameMatcher {
r := regex.MustCompile(pattern)
return func(name string) bool {
return r.MatchString(name)
}
}
func nameContains(pattern string) GeneratedCodeNameMatcher {
return func(name string) bool {
return strings.Contains(name, pattern)
}
}
func nameEndsWith(pattern string) GeneratedCodeNameMatcher {
return func(name string) bool {
return strings.HasSuffix(name, pattern)
}
}
// GeneratedCodeNameMatchers are all the matchers that check whether the code
// is generated based only on the file name.
var GeneratedCodeNameMatchers = []GeneratedCodeNameMatcher{
// Cocoa pods
nameMatches(`(^Pods|\/Pods)\/`),
// Carthage build
nameMatches(`(^|\/)Carthage\/Build\/`),
// NET designer file
nameMatches(`(?i)\.designer\.(cs|vb)$`),
// Generated NET specflow feature file
nameEndsWith(".feature.cs"),
// Node modules
nameContains("node_modules/"),
// Go vendor
nameMatches(`vendor\/([-0-9A-Za-z]+\.)+(com|edu|gov|in|me|net|org|fm|io)`),
// Go lock
nameEndsWith("Gopkg.lock"),
nameEndsWith("glide.lock"),
// Esy lock
nameMatches(`(^|\/)(\w+\.)?esy.lock$`),
// NPM shrinkwrap
nameEndsWith("npm-shrinkwrap.json"),
// NPM package lock
nameEndsWith("package-lock.json"),
// Yarn plugnplay
nameMatches(`(^|\/)\.pnp\..*$`),
// Godeps
nameContains("Godeps/"),
// Composer lock
nameEndsWith("composer.lock"),
// Generated by zephir
nameMatches(`.\.zep\.(?:c|h|php)$`),
// Cargo lock
nameEndsWith("Cargo.lock"),
// Pipenv lock
nameEndsWith("Pipfile.lock"),
// GraphQL relay
nameContains("__generated__/"),
// Poetry lock
nameEndsWith("poetry.lock"),
}
// GeneratedCodeMatcher checks whether the file with the given data is
// generated code.
type GeneratedCodeMatcher func(path, ext string, content []byte) bool
// GeneratedCodeMatchers is the list of all generated code matchers that
// rely on checking the content of the file to make the guess.
var GeneratedCodeMatchers = []GeneratedCodeMatcher{
isMinifiedFile,
hasSourceMapReference,
isSourceMap,
isCompiledCoffeeScript,
isGeneratedNetDocfile,
isGeneratedJavaScriptPEGParser,
isGeneratedPostScript,
isGeneratedGo,
isGeneratedProtobufFromGo,
isGeneratedProtobuf,
isGeneratedJavaScriptProtocolBuffer,
isGeneratedApacheThrift,
isGeneratedJNIHeader,
isVCRCassette,
isCompiledCythonFile,
isGeneratedModule,
isGeneratedUnity3DMeta,
isGeneratedRacc,
isGeneratedJFlex,
isGeneratedGrammarKit,
isGeneratedRoxygen2,
isGeneratedJison,
isGeneratedGRPCCpp,
isGeneratedDart,
isGeneratedPerlPPPortHeader,
isGeneratedGameMakerStudio,
isGeneratedGimp,
isGeneratedVisualStudio6,
isGeneratedHaxe,
isGeneratedHTML,
isGeneratedJooq,
}
func canBeMinified(ext string) bool {
return ext == ".js" || ext == ".css"
}
// isMinifiedFile returns whether the file may be minified.
// We consider a minified file any css or js file whose average number of chars
// per line is more than 110.
func isMinifiedFile(path, ext string, content []byte) bool {
if !canBeMinified(ext) {
return false
}
var chars, lines uint64
forEachLine(content, func(line []byte) {
chars += uint64(len(line))
lines++
})
if lines == 0 {
return false
}
return chars/lines > 110
}
var sourceMapRegex = regex.MustCompile(`^\/[*\/][\#@] source(?:Mapping)?URL|sourceURL=`)
// hasSourceMapReference returns whether the file contains a reference to a
// source-map file.
func hasSourceMapReference(_ string, ext string, content []byte) bool {
if !canBeMinified(ext) {
return false
}
for _, line := range getLines(content, -2) {
if sourceMapRegex.Match(line) {
return true
}
}
return false
}
var sourceMapRegexps = []regex.EnryRegexp{
regex.MustCompile(`^{"version":\d+,`),
regex.MustCompile(`^\/\*\* Begin line maps\. \*\*\/{`),
}
// isSourceMap returns whether the file itself is a source map.
func isSourceMap(path, _ string, content []byte) bool {
if strings.HasSuffix(path, ".js.map") || strings.HasSuffix(path, ".css.map") {
return true
}
firstLine := getFirstLine(content)
if len(firstLine) == 0 {
return false
}
for _, r := range sourceMapRegexps {
if r.Match(firstLine) {
return true
}
}
return false
}
func isCompiledCoffeeScript(path, ext string, content []byte) bool {
if ext != ".js" {
return false
}
firstLine := getFirstLine(content)
lastLines := getLines(content, -2)
if len(lastLines) < 2 {
return false
}
if string(firstLine) == "(function() {" &&
string(lastLines[1]) == "}).call(this);" &&
string(lastLines[0]) == "" {
score := 0
forEachLine(content, func(line []byte) {
if bytes.Contains(line, []byte("var ")) {
// Underscored temp vars are likely to be Coffee
score += 1 * countAppearancesInLine(line, "_fn", "_i", "_len", "_ref", "_results")
// bind and extend functions are very Coffee specific
score += 3 * countAppearancesInLine(line, "__bind", "__extends", "__hasProp", "__indexOf", "__slice")
}
})
// Require a score of 3. This is fairly arbitrary. Consider tweaking later.
// See: https://github.com/github/linguist/blob/master/lib/linguist/generated.rb#L176-L213
return score >= 3
}
return false
}
func isGeneratedNetDocfile(_, ext string, content []byte) bool {
if ext != ".xml" {
return false
}
lines := bytes.Split(content, []byte{'\n'})
if len(lines) <= 3 {
return false
}
return bytes.Contains(lines[1], []byte("<doc>")) &&
bytes.Contains(lines[2], []byte("<assembly>")) &&
bytes.Contains(lines[len(lines)-2], []byte("</doc>"))
}
var pegJavaScriptGeneratedRegex = regex.MustCompile(`^(?:[^\/]|\/[^\*])*\/\*(?:[^\*]|\*[^\/])*Generated by PEG.js`)
func isGeneratedJavaScriptPEGParser(_, ext string, content []byte) bool {
if ext != ".js" {
return false
}
// PEG.js-generated parsers include a comment near the top of the file
// that marks them as such.
return pegJavaScriptGeneratedRegex.Match(bytes.Join(getLines(content, 5), []byte("")))
}
var postScriptType1And42Regex = regex.MustCompile(`(\n|\r\n|\r)\s*(?:currentfile eexec\s+|\/sfnts\s+\[)`)
var postScriptRegexes = []regex.EnryRegexp{
regex.MustCompile(`[0-9]|draw|mpage|ImageMagick|inkscape|MATLAB`),
regex.MustCompile(`PCBNEW|pnmtops|\(Unknown\)|Serif Affinity|Filterimage -tops`),
}
func isGeneratedPostScript(_, ext string, content []byte) bool {
if ext != ".ps" && ext != ".eps" && ext != ".pfa" {
return false
}
// Type 1 and Type 42 fonts converted to PostScript are stored as hex-encoded byte streams; these
// streams are always preceded the `eexec` operator (if Type 1), or the `/sfnts` key (if Type 42).
if postScriptType1And42Regex.Match(content) {
return true
}
// We analyze the "%%Creator:" comment, which contains the author/generator
// of the file. If there is one, it should be in one of the first few lines.
var creator []byte
for _, line := range getLines(content, 10) {
if bytes.HasPrefix(line, []byte("%%Creator: ")) {
creator = line
break
}
}
if len(creator) == 0 {
return false
}
// EAGLE doesn't include a version number when it generates PostScript.
// However, it does prepend its name to the document's "%%Title" field.
if bytes.Contains(creator, []byte("EAGLE")) {
for _, line := range getLines(content, 5) {
if bytes.HasPrefix(line, []byte("%%Title: EAGLE Drawing ")) {
return true
}
}
}
// Most generators write their version number, while human authors' or companies'
// names don't contain numbers. So look if the line contains digits. Also
// look for some special cases without version numbers.
for _, r := range postScriptRegexes {
if r.Match(creator) {
return true
}
}
return false
}
func isGeneratedGo(_, ext string, content []byte) bool {
if ext != ".go" {
return false
}
lines := getLines(content, 40)
if len(lines) <= 1 {
return false
}
for _, line := range lines {
if bytes.Contains(line, []byte("Code generated by")) {
return true
}
}
return false
}
func isGeneratedProtobufFromGo(_, ext string, content []byte) bool {
if ext != ".proto" {
return false
}
lines := getLines(content, 20)
if len(lines) <= 1 {
return false
}
for _, line := range lines {
if bytes.Contains(line, []byte("This file was autogenerated by go-to-protobuf")) {
return true
}
}
return false
}
var protoExtensions = map[string]struct{}{
".py": {},
".java": {},
".h": {},
".cc": {},
".cpp": {},
".m": {},
".rb": {},
".php": {},
}
func isGeneratedProtobuf(_, ext string, content []byte) bool {
if _, ok := protoExtensions[ext]; !ok {
return false
}
lines := getLines(content, 3)
if len(lines) <= 1 {
return false
}
for _, line := range lines {
if bytes.Contains(line, []byte("Generated by the protocol buffer compiler. DO NOT EDIT!")) {
return true
}
}
return false
}
func isGeneratedJavaScriptProtocolBuffer(_, ext string, content []byte) bool {
if ext != ".js" {
return false
}
lines := getLines(content, 6)
if len(lines) < 6 {
return false
}
return bytes.Contains(lines[5], []byte("GENERATED CODE -- DO NOT EDIT!"))
}
var apacheThriftExtensions = map[string]struct{}{
".rb": {},
".py": {},
".go": {},
".js": {},
".m": {},
".java": {},
".h": {},
".cc": {},
".cpp": {},
".php": {},
}
func isGeneratedApacheThrift(_, ext string, content []byte) bool {
if _, ok := apacheThriftExtensions[ext]; !ok {
return false
}
for _, line := range getLines(content, 6) {
if bytes.Contains(line, []byte("Autogenerated by Thrift Compiler")) {
return true
}
}
return false
}
func isGeneratedJNIHeader(_, ext string, content []byte) bool {
if ext != ".h" {
return false
}
lines := getLines(content, 2)
if len(lines) < 2 {
return false
}
return bytes.Contains(lines[0], []byte("/* DO NOT EDIT THIS FILE - it is machine generated */")) &&
bytes.Contains(lines[1], []byte("#include <jni.h>"))
}
func isVCRCassette(_, ext string, content []byte) bool {
if ext != ".yml" {
return false
}
lines := getLines(content, -2)
if len(lines) < 2 {
return false
}
return bytes.Contains(lines[1], []byte("recorded_with: VCR"))
}
func isCompiledCythonFile(_, ext string, content []byte) bool {
if ext != ".c" && ext != ".cpp" {
return false
}
lines := getLines(content, 1)
if len(lines) < 1 {
return false
}
return bytes.Contains(lines[0], []byte("Generated by Cython"))
}
func isGeneratedModule(_, ext string, content []byte) bool {
if ext != ".mod" {
return false
}
lines := getLines(content, 1)
if len(lines) < 1 {
return false
}
return bytes.Contains(lines[0], []byte("PCBNEW-LibModule-V")) ||
bytes.Contains(lines[0], []byte("GFORTRAN module version '"))
}
func isGeneratedUnity3DMeta(_, ext string, content []byte) bool {
if ext != ".meta" {
return false
}
lines := getLines(content, 1)
if len(lines) < 1 {
return false
}
return bytes.Contains(lines[0], []byte("fileFormatVersion: "))
}
func isGeneratedRacc(_, ext string, content []byte) bool {
if ext != ".rb" {
return false
}
lines := getLines(content, 3)
if len(lines) < 3 {
return false
}
return bytes.HasPrefix(lines[2], []byte("# This file is automatically generated by Racc"))
}
func isGeneratedJFlex(_, ext string, content []byte) bool {
if ext != ".java" {
return false
}
lines := getLines(content, 1)
if len(lines) < 1 {
return false
}
return bytes.HasPrefix(lines[0], []byte("/* The following code was generated by JFlex "))
}
func isGeneratedGrammarKit(_, ext string, content []byte) bool {
if ext != ".java" {
return false
}
lines := getLines(content, 1)
if len(lines) < 1 {
return false
}
return bytes.Contains(lines[0], []byte("// This is a generated file. Not intended for manual editing."))
}
func isGeneratedRoxygen2(_, ext string, content []byte) bool {
if ext != ".rd" {
return false
}
lines := getLines(content, 1)
if len(lines) < 1 {
return false
}
return bytes.Contains(lines[0], []byte("% Generated by roxygen2: do not edit by hand"))
}
func isGeneratedJison(_, ext string, content []byte) bool {
if ext != ".js" {
return false
}
lines := getLines(content, 1)
if len(lines) < 1 {
return false
}
return bytes.Contains(lines[0], []byte("/* parser generated by jison ")) ||
bytes.Contains(lines[0], []byte("/* generated by jison-lex "))
}
func isGeneratedGRPCCpp(_, ext string, content []byte) bool {
switch ext {
case ".cpp", ".hpp", ".h", ".cc":
lines := getLines(content, 1)
if len(lines) < 1 {
return false
}
return bytes.Contains(lines[0], []byte("// Generated by the gRPC"))
default:
return false
}
}
var dartRegex = regex.MustCompile(`generated code\W{2,3}do not modify`)
func isGeneratedDart(_, ext string, content []byte) bool {
if ext != ".dart" {
return false
}
lines := getLines(content, 1)
if len(lines) < 1 {
return false
}
return dartRegex.Match(bytes.ToLower(lines[0]))
}
func isGeneratedPerlPPPortHeader(name, _ string, content []byte) bool {
if !strings.HasSuffix(name, "ppport.h") {
return false
}
lines := getLines(content, 10)
if len(lines) < 10 {
return false
}
return bytes.Contains(lines[8], []byte("Automatically created by Devel::PPPort"))
}
var (
gameMakerStudioFirstLineRegex = regex.MustCompile(`^\d\.\d\.\d.+\|\{`)
gameMakerStudioThirdLineRegex = regex.MustCompile(`\"modelName\"\:\s*\"GM`)
)
func isGeneratedGameMakerStudio(_, ext string, content []byte) bool {
if ext != ".yy" && ext != ".yyp" {
return false
}
lines := getLines(content, 3)
if len(lines) < 3 {
return false
}
return gameMakerStudioThirdLineRegex.Match(lines[2]) ||
gameMakerStudioFirstLineRegex.Match(lines[0])
}
var gimpRegexes = []regex.EnryRegexp{
regex.MustCompile(`\/\* GIMP [a-zA-Z0-9\- ]+ C\-Source image dump \(.+?\.c\) \*\/`),
regex.MustCompile(`\/\* GIMP header image file format \([a-zA-Z0-9\- ]+\)\: .+?\.h \*\/`),
}
func isGeneratedGimp(_, ext string, content []byte) bool {
if ext != ".c" && ext != ".h" {
return false
}
lines := getLines(content, 1)
if len(lines) < 1 {
return false
}
for _, r := range gimpRegexes {
if r.Match(lines[0]) {
return true
}
}
return false
}
func isGeneratedVisualStudio6(_, ext string, content []byte) bool {
if ext != ".dsp" {
return false
}
for _, l := range getLines(content, 3) {
if bytes.Contains(l, []byte("# Microsoft Developer Studio Generated Build File")) {
return true
}
}
return false
}
var haxeExtensions = map[string]struct{}{
".js": {},
".py": {},
".lua": {},
".cpp": {},
".h": {},
".java": {},
".cs": {},
".php": {},
}
func isGeneratedHaxe(_, ext string, content []byte) bool {
if _, ok := haxeExtensions[ext]; !ok {
return false
}
for _, l := range getLines(content, 3) {
if bytes.Contains(l, []byte("Generated by Haxe")) {
return true
}
}
return false
}
var (
doxygenRegex = regex.MustCompile(`<!--\s+Generated by Doxygen\s+[.0-9]+\s*-->`)
htmlMetaRegex = regex.MustCompile(`<meta(\s+[^>]+)>`)
htmlMetaContentRegex = regex.MustCompile(`\s+(name|content|value)\s*=\s*("[^"]+"|'[^']+'|[^\s"']+)`)
orgModeMetaRegex = regex.MustCompile(`org\s+mode`)
)
func isGeneratedHTML(_, ext string, content []byte) bool {
if ext != ".html" && ext != ".htm" && ext != ".xhtml" {
return false
}
lines := getLines(content, 30)
// Pkgdown
if len(lines) >= 2 {
for _, l := range lines[:2] {
if bytes.Contains(l, []byte("<!-- Generated by pkgdown: do not edit by hand -->")) {
return true
}
}
}
// Mandoc
if len(lines) > 2 &&
bytes.HasPrefix(lines[2], []byte("<!-- This is an automatically generated file.")) {
return true
}
// Doxygen
for _, l := range lines {
if doxygenRegex.Match(l) {
return true
}
}
// HTML tag: <meta name="generator" content="" />
part := bytes.ToLower(bytes.Join(lines, []byte{' '}))
part = bytes.ReplaceAll(part, []byte{'\n'}, []byte{})
part = bytes.ReplaceAll(part, []byte{'\r'}, []byte{})
matches := htmlMetaRegex.FindAll(part, -1)
if len(matches) == 0 {
return false
}
for _, m := range matches {
var name, value, content string
ms := htmlMetaContentRegex.FindAllStringSubmatch(string(m), -1)
for _, m := range ms {
switch m[1] {
case "name":
name = m[2]
case "value":
value = m[2]
case "content":
content = m[2]
}
}
var val = value
if val == "" {
val = content
}
name = strings.Trim(name, `"'`)
val = strings.Trim(val, `"'`)
if name != "generator" || val == "" {
continue
}
if strings.Contains(val, "jlatex2html") ||
strings.Contains(val, "latex2html") ||
strings.Contains(val, "groff") ||
strings.Contains(val, "makeinfo") ||
strings.Contains(val, "texi2html") ||
strings.Contains(val, "ronn") ||
orgModeMetaRegex.MatchString(val) {
return true
}
}
return false
}
func isGeneratedJooq(_, ext string, content []byte) bool {
if ext != ".java" {
return false
}
for _, l := range getLines(content, 2) {
if bytes.Contains(l, []byte("This file is generated by jOOQ.")) {
return true
}
}
return false
}
func getFirstLine(content []byte) []byte {
lines := getLines(content, 1)
if len(lines) > 0 {
return lines[0]
}
return nil
}
// getLines returns up to the first n lines. A negative index will return up to
// the last n lines in reverse order.
func getLines(content []byte, n int) [][]byte {
var result [][]byte
if n < 0 {
for pos := len(content); pos > 0 && len(result) < -n; {
nlpos := bytes.LastIndexByte(content[:pos], '\n')
if nlpos+1 < len(content)-1 {
result = append(result, content[nlpos+1:pos])
}
pos = nlpos
}
} else {
for pos := 0; pos < len(content) && len(result) < n; {
nlpos := bytes.IndexByte(content[pos:], '\n')
if nlpos < 0 && pos < len(content) {
nlpos = len(content)
} else if nlpos >= 0 {
nlpos += pos
}
result = append(result, content[pos:nlpos])
pos = nlpos + 1
}
}
return result
}
func forEachLine(content []byte, cb func([]byte)) {
var pos int
for pos < len(content) {
nlpos := bytes.IndexByte(content[pos:], '\n')
if nlpos < 0 && pos < len(content) {
nlpos = len(content)
} else if nlpos >= 0 {
nlpos += pos
}
cb(content[pos:nlpos])
pos = nlpos + 1
}
}
func countAppearancesInLine(line []byte, targets ...string) int {
var count int
for _, t := range targets {
count += bytes.Count(line, []byte(t))
}
return count
}

54
data/generated_test.go Normal file
View File

@ -0,0 +1,54 @@
package data
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestGetFirstLineEmptyContent(t *testing.T) {
require.Nil(t, getFirstLine(nil))
}
func TestForEachLine(t *testing.T) {
const sample = "foo\nbar\nboomboom\nbleepbloop\n"
var lines = strings.Split(sample, "\n")
var result []string
forEachLine([]byte(sample), func(l []byte) {
result = append(result, string(l))
})
require.Equal(t, lines[:len(lines)-1], result)
}
func TestGetLines(t *testing.T) {
const sample = "foo\nbar\nboomboom\nbleepbloop\n"
testCases := []struct {
lines int
expected []string
}{
{1, []string{"foo"}},
{2, []string{"foo", "bar"}},
{10, []string{"foo", "bar", "boomboom", "bleepbloop"}},
{-1, []string{"bleepbloop"}},
{-2, []string{"bleepbloop", "boomboom"}},
{-10, []string{"bleepbloop", "boomboom", "bar", "foo"}},
}
for _, tt := range testCases {
t.Run(fmt.Sprint(tt.lines), func(t *testing.T) {
lines := getLines([]byte(sample), tt.lines)
var result = make([]string, len(lines))
for i, l := range lines {
result[i] = string(l)
}
require.Equal(t, tt.expected, result)
})
}
}

87
data/groups.go Normal file
View File

@ -0,0 +1,87 @@
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 42fd3c2514375fc6ad281552368edd3fb9f6ee7f
package data
var LanguagesGroup = map[string]string{
"Alpine Abuild": "Shell",
"Apollo Guidance Computer": "Assembly",
"BibTeX": "TeX",
"Bison": "Yacc",
"Bluespec BH": "Bluespec",
"C2hs Haskell": "Haskell",
"CameLIGO": "LigoLANG",
"ColdFusion CFC": "ColdFusion",
"ECLiPSe": "Prolog",
"Easybuild": "Python",
"Ecere Projects": "JavaScript",
"Ecmarkup": "HTML",
"EditorConfig": "INI",
"Elvish Transcript": "Elvish",
"Filterscript": "RenderScript",
"Fortran": "Fortran",
"Fortran Free Form": "Fortran",
"Gentoo Ebuild": "Shell",
"Gentoo Eclass": "Shell",
"Git Config": "INI",
"Glimmer JS": "JavaScript",
"Glimmer TS": "TypeScript",
"Gradle Kotlin DSL": "Gradle",
"Groovy Server Pages": "Groovy",
"HTML+ECR": "HTML",
"HTML+EEX": "HTML",
"HTML+ERB": "HTML",
"HTML+PHP": "HTML",
"HTML+Razor": "HTML",
"Isabelle ROOT": "Isabelle",
"JFlex": "Lex",
"JSON with Comments": "JSON",
"Java Server Pages": "Java",
"JavaScript+ERB": "JavaScript",
"Jison": "Yacc",
"Jison Lex": "Lex",
"Julia REPL": "Julia",
"Lean 4": "Lean",
"LigoLANG": "LigoLANG",
"Literate Agda": "Agda",
"Literate CoffeeScript": "CoffeeScript",
"Literate Haskell": "Haskell",
"M4Sugar": "M4",
"MUF": "Forth",
"Maven POM": "XML",
"Motorola 68K Assembly": "Assembly",
"NPM Config": "INI",
"NumPy": "Python",
"OASv2-json": "OpenAPI Specification v2",
"OASv2-yaml": "OpenAPI Specification v2",
"OASv3-json": "OpenAPI Specification v3",
"OASv3-yaml": "OpenAPI Specification v3",
"OpenCL": "C",
"OpenRC runscript": "Shell",
"Parrot Assembly": "Parrot",
"Parrot Internal Representation": "Parrot",
"Pic": "Roff",
"PostCSS": "CSS",
"Python console": "Python",
"Python traceback": "Python",
"RBS": "Ruby",
"Readline Config": "INI",
"ReasonLIGO": "LigoLANG",
"Roff Manpage": "Roff",
"SSH Config": "INI",
"STON": "Smalltalk",
"Simple File Verification": "Checksums",
"Snakemake": "Python",
"TSX": "TypeScript",
"Tcsh": "Shell",
"Terraform Template": "HCL",
"Unified Parallel C": "C",
"Unix Assembly": "Assembly",
"Wget Config": "INI",
"X BitMap": "C",
"X PixMap": "C",
"XML Property List": "XML",
"cURL Config": "INI",
"fish": "Shell",
"nanorc": "INI",
}

35
data/heuristics.go Normal file
View File

@ -0,0 +1,35 @@
package data
import "github.com/go-enry/go-enry/v2/data/rule"
// Heuristics implements a rule-based content matching engine.
// Heuristics is a number of sequentially applied rule.Heuristic where a
// matching one disambiguates language(s) for a single file extension.
type Heuristics []rule.Heuristic
// Match returns languages identified by the matching rule of the heuristic.
func (hs Heuristics) Match(data []byte) []string {
var matchedLangs []string
for _, heuristic := range hs {
if heuristic.Match(data) {
for _, langOrAlias := range heuristic.Languages() {
lang, ok := LanguageByAlias(langOrAlias)
if !ok { // should never happen
// reaching here means language name/alias in heuristics.yml
// is not consistent with languages.yml
// but we do not surface any such error at the API
continue
}
matchedLangs = append(matchedLangs, lang)
}
break
}
}
return matchedLangs
}
// matchString is a convenience used only in tests.
func (hs *Heuristics) matchString(data string) []string {
return hs.Match([]byte(data))
}

61
data/heuristics_test.go Normal file
View File

@ -0,0 +1,61 @@
package data
import (
"regexp"
"testing"
"github.com/go-enry/go-enry/v2/data/rule"
"github.com/stretchr/testify/assert"
)
var testContentHeuristics = map[string]*Heuristics{
".md": &Heuristics{ // final pattern for parsed YAML rule
rule.Or(
rule.MatchingLanguages("Markdown"),
regexp.MustCompile(`(^[-A-Za-z0-9=#!\*\[|>])|<\/ | \A\z`),
),
rule.Or(
rule.MatchingLanguages("GCC Machine Description"),
regexp.MustCompile(`^(;;|\(define_)`),
),
rule.Always(
rule.MatchingLanguages("Markdown"),
),
},
".ms": &Heuristics{
// Order defines precedence: And, Or, Not, Named, Always
rule.And(
rule.MatchingLanguages("Unix Assembly"),
rule.Not(rule.MatchingLanguages(""), regexp.MustCompile(`/\*`)),
rule.Or(
rule.MatchingLanguages(""),
regexp.MustCompile(`^\s*\.(?:include\s|globa?l\s|[A-Za-z][_A-Za-z0-9]*:)`),
),
),
rule.Or(
rule.MatchingLanguages("Roff"),
regexp.MustCompile(`^[.''][A-Za-z]{2}(\s|$)`),
),
rule.Always(
rule.MatchingLanguages("MAXScript"),
),
},
}
func TestContentHeuristic_MatchingAlways(t *testing.T) {
lang := testContentHeuristics[".md"].matchString("")
assert.Equal(t, []string{"Markdown"}, lang)
lang = testContentHeuristics[".ms"].matchString("")
assert.Equal(t, []string{"MAXScript"}, lang)
}
func TestContentHeuristic_MatchingAnd(t *testing.T) {
lang := testContentHeuristics[".md"].matchString(";;")
assert.Equal(t, []string{"GCC Machine Description"}, lang)
}
func TestContentHeuristic_MatchingOr(t *testing.T) {
lang := testContentHeuristics[".ms"].matchString(" .include \"math.s\"")
assert.Equal(t, []string{"Unix Assembly"}, lang)
}

737
data/id.go Normal file
View File

@ -0,0 +1,737 @@
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 42fd3c2514375fc6ad281552368edd3fb9f6ee7f
package data
var IDByLanguage = map[string]int{
"1C Enterprise": 0,
"2-Dimensional Array": 387204628,
"4D": 577529595,
"ABAP": 1,
"ABAP CDS": 452681853,
"ABNF": 429,
"AGS Script": 2,
"AIDL": 451700185,
"AL": 658971832,
"AMPL": 3,
"ANTLR": 4,
"API Blueprint": 5,
"APL": 6,
"ASL": 124996147,
"ASN.1": 7,
"ASP.NET": 564186416,
"ATS": 9,
"ActionScript": 10,
"Ada": 11,
"Adblock Filter List": 884614762,
"Adobe Font Metrics": 147198098,
"Agda": 12,
"Alloy": 13,
"Alpine Abuild": 14,
"Altium Designer": 187772328,
"AngelScript": 389477596,
"Ant Build System": 15,
"Antlers": 1067292663,
"ApacheConf": 16,
"Apex": 17,
"Apollo Guidance Computer": 18,
"AppleScript": 19,
"Arc": 20,
"AsciiDoc": 22,
"AspectJ": 23,
"Assembly": 24,
"Astro": 578209015,
"Asymptote": 591605007,
"Augeas": 25,
"AutoHotkey": 26,
"AutoIt": 27,
"Avro IDL": 785497837,
"Awk": 28,
"BASIC": 28923963,
"BQN": 330386870,
"Ballerina": 720859680,
"Batchfile": 29,
"Beef": 545626333,
"Befunge": 30,
"Berry": 121855308,
"BibTeX": 982188347,
"Bicep": 321200902,
"Bikeshed": 1055528081,
"Bison": 31,
"BitBake": 32,
"Blade": 33,
"BlitzBasic": 34,
"BlitzMax": 35,
"Bluespec": 36,
"Bluespec BH": 641580358,
"Boo": 37,
"Boogie": 955017407,
"Brainfuck": 38,
"BrighterScript": 943571030,
"Brightscript": 39,
"Browserslist": 153503348,
"C": 41,
"C#": 42,
"C++": 43,
"C-ObjDump": 44,
"C2hs Haskell": 45,
"CAP CDS": 390788699,
"CIL": 29176339,
"CLIPS": 46,
"CMake": 47,
"COBOL": 48,
"CODEOWNERS": 321684729,
"COLLADA": 49,
"CSON": 424,
"CSS": 50,
"CSV": 51,
"CUE": 356063509,
"CWeb": 657332628,
"Cabal Config": 677095381,
"Caddyfile": 615465151,
"Cadence": 270184138,
"Cairo": 620599567,
"CameLIGO": 829207807,
"Cap'n Proto": 52,
"CartoCSS": 53,
"Ceylon": 54,
"Chapel": 55,
"Charity": 56,
"Checksums": 372063053,
"ChucK": 57,
"Circom": 1042332086,
"Cirru": 58,
"Clarion": 59,
"Clarity": 91493841,
"Classic ASP": 8,
"Clean": 60,
"Click": 61,
"Clojure": 62,
"Closure Templates": 357046146,
"Cloud Firestore Security Rules": 407996372,
"CoNLL-U": 421026389,
"CodeQL": 424259634,
"CoffeeScript": 63,
"ColdFusion": 64,
"ColdFusion CFC": 65,
"Common Lisp": 66,
"Common Workflow Language": 988547172,
"Component Pascal": 67,
"Cool": 68,
"Coq": 69,
"Cpp-ObjDump": 70,
"Creole": 71,
"Crystal": 72,
"Csound": 73,
"Csound Document": 74,
"Csound Score": 75,
"Cuda": 77,
"Cue Sheet": 942714150,
"Curry": 439829048,
"Cycript": 78,
"Cypher": 850806976,
"Cython": 79,
"D": 80,
"D-ObjDump": 81,
"D2": 37531557,
"DIGITAL Command Language": 82,
"DM": 83,
"DNS Zone": 84,
"DTrace": 85,
"Dafny": 969323346,
"Darcs Patch": 86,
"Dart": 87,
"DataWeave": 974514097,
"Debian Package Control File": 527438264,
"DenizenScript": 435000929,
"Dhall": 793969321,
"Diff": 88,
"DirectX 3D File": 201049282,
"Dockerfile": 89,
"Dogescript": 90,
"Dotenv": 111148035,
"Dylan": 91,
"E": 92,
"E-mail": 529653389,
"EBNF": 430,
"ECL": 93,
"ECLiPSe": 94,
"EJS": 95,
"EQ": 96,
"Eagle": 97,
"Earthly": 963512632,
"Easybuild": 342840477,
"Ecere Projects": 98,
"Ecmarkup": 844766630,
"Edge": 460509620,
"EdgeQL": 925235833,
"EditorConfig": 96139566,
"Edje Data Collection": 342840478,
"Eiffel": 99,
"Elixir": 100,
"Elm": 101,
"Elvish": 570996448,
"Elvish Transcript": 452025714,
"Emacs Lisp": 102,
"EmberScript": 103,
"Erlang": 104,
"Euphoria": 880693982,
"F#": 105,
"F*": 336943375,
"FIGlet Font": 686129783,
"FIRRTL": 906694254,
"FLUX": 106,
"Factor": 108,
"Fancy": 109,
"Fantom": 110,
"Faust": 622529198,
"Fennel": 239946126,
"Filebench WML": 111,
"Filterscript": 112,
"Fluent": 206353404,
"Formatted": 113,
"Forth": 114,
"Fortran": 107,
"Fortran Free Form": 761352333,
"FreeBasic": 472896659,
"FreeMarker": 115,
"Frege": 116,
"Futhark": 97358117,
"G-code": 117,
"GAML": 290345951,
"GAMS": 118,
"GAP": 119,
"GCC Machine Description": 121,
"GDB": 122,
"GDScript": 123,
"GEDCOM": 459577965,
"GLSL": 124,
"GN": 302957008,
"GSC": 257856279,
"Game Maker Language": 125,
"Gemfile.lock": 907065713,
"Gemini": 310828396,
"Genero 4gl": 986054050,
"Genero per": 902995658,
"Genie": 792408528,
"Genshi": 126,
"Gentoo Ebuild": 127,
"Gentoo Eclass": 128,
"Gerber Image": 404627610,
"Gettext Catalog": 129,
"Gherkin": 76,
"Git Attributes": 956324166,
"Git Config": 807968997,
"Git Revision List": 461881235,
"Gleam": 1054258749,
"Glimmer JS": 5523150,
"Glimmer TS": 95110458,
"Glyph": 130,
"Glyph Bitmap Distribution Format": 997665271,
"Gnuplot": 131,
"Go": 132,
"Go Checksums": 1054391671,
"Go Module": 947461016,
"Go Workspace": 934546256,
"Godot Resource": 738107771,
"Golo": 133,
"Gosu": 134,
"Grace": 135,
"Gradle": 136,
"Gradle Kotlin DSL": 432600901,
"Grammatical Framework": 137,
"Graph Modeling Language": 138,
"GraphQL": 139,
"Graphviz (DOT)": 140,
"Groovy": 142,
"Groovy Server Pages": 143,
"HAProxy": 366607477,
"HCL": 144,
"HLSL": 145,
"HOCON": 679725279,
"HTML": 146,
"HTML+ECR": 148,
"HTML+EEX": 149,
"HTML+ERB": 150,
"HTML+PHP": 151,
"HTML+Razor": 479039817,
"HTTP": 152,
"HXML": 786683730,
"Hack": 153,
"Haml": 154,
"Handlebars": 155,
"Harbour": 156,
"Haskell": 157,
"Haxe": 158,
"HiveQL": 931814087,
"HolyC": 928121743,
"Hosts File": 231021894,
"Hy": 159,
"HyPhy": 160,
"IDL": 161,
"IGOR Pro": 162,
"INI": 163,
"IRC log": 164,
"Idris": 165,
"Ignore List": 74444240,
"ImageJ Macro": 575143428,
"Imba": 1057618448,
"Inform 7": 166,
"Ink": 838252715,
"Inno Setup": 167,
"Io": 168,
"Ioke": 169,
"Isabelle": 170,
"Isabelle ROOT": 171,
"J": 172,
"JAR Manifest": 447261135,
"JCL": 316620079,
"JFlex": 173,
"JSON": 174,
"JSON with Comments": 423,
"JSON5": 175,
"JSONLD": 176,
"JSONiq": 177,
"Janet": 1028705371,
"Jasmin": 180,
"Java": 181,
"Java Properties": 519377561,
"Java Server Pages": 182,
"JavaScript": 183,
"JavaScript+ERB": 914318960,
"Jest Snapshot": 774635084,
"JetBrains MPS": 465165328,
"Jinja": 147,
"Jison": 284531423,
"Jison Lex": 406395330,
"Jolie": 998078858,
"Jsonnet": 664885656,
"Julia": 184,
"Julia REPL": 220689142,
"Jupyter Notebook": 185,
"Just": 128447695,
"KRL": 186,
"Kaitai Struct": 818804755,
"KakouneScript": 603336474,
"KerboScript": 59716426,
"KiCad Layout": 187,
"KiCad Legacy Layout": 140848857,
"KiCad Schematic": 622447435,
"Kickstart": 692635484,
"Kit": 188,
"Kotlin": 189,
"Kusto": 225697190,
"LFE": 190,
"LLVM": 191,
"LOLCODE": 192,
"LSL": 193,
"LTspice Symbol": 1013566805,
"LabVIEW": 194,
"Lark": 758480799,
"Lasso": 195,
"Latte": 196,
"Lean": 197,
"Lean 4": 455147478,
"Less": 198,
"Lex": 199,
"LigoLANG": 1040646257,
"LilyPond": 200,
"Limbo": 201,
"Linker Script": 202,
"Linux Kernel Module": 203,
"Liquid": 204,
"Literate Agda": 205,
"Literate CoffeeScript": 206,
"Literate Haskell": 207,
"LiveScript": 208,
"Logos": 209,
"Logtalk": 210,
"LookML": 211,
"LoomScript": 212,
"Lua": 213,
"Luau": 365050359,
"M": 214,
"M4": 215,
"M4Sugar": 216,
"MATLAB": 225,
"MAXScript": 217,
"MDX": 512838272,
"MLIR": 448253929,
"MQL4": 426,
"MQL5": 427,
"MTML": 218,
"MUF": 219,
"Macaulay2": 34167825,
"Makefile": 220,
"Mako": 221,
"Markdown": 222,
"Marko": 932782397,
"Mask": 223,
"Mathematica": 224,
"Maven POM": 226,
"Max": 227,
"Mercury": 229,
"Mermaid": 385992043,
"Meson": 799141244,
"Metal": 230,
"Microsoft Developer Studio Project": 800983837,
"Microsoft Visual Studio Solution": 849523096,
"MiniD": 231,
"MiniYAML": 4896465,
"Mint": 968740319,
"Mirah": 232,
"Modelica": 233,
"Modula-2": 234,
"Modula-3": 564743864,
"Module Management System": 235,
"Mojo": 1045019587,
"Monkey": 236,
"Monkey C": 231751931,
"Moocode": 237,
"MoonScript": 238,
"Motoko": 202937027,
"Motorola 68K Assembly": 477582706,
"Move": 638334599,
"Muse": 474864066,
"Mustache": 638334590,
"Myghty": 239,
"NASL": 171666519,
"NCL": 240,
"NEON": 481192983,
"NL": 241,
"NMODL": 136456478,
"NPM Config": 685022663,
"NSIS": 242,
"NWScript": 731233819,
"Nasal": 178322513,
"Nearley": 521429430,
"Nemerle": 243,
"NetLinx": 244,
"NetLinx+ERB": 245,
"NetLogo": 246,
"NewLisp": 247,
"Nextflow": 506780613,
"Nginx": 248,
"Nim": 249,
"Ninja": 250,
"Nit": 251,
"Nix": 252,
"Nu": 253,
"NumPy": 254,
"Nunjucks": 461856962,
"Nushell": 446573572,
"OASv2-json": 834374816,
"OASv2-yaml": 105187618,
"OASv3-json": 980062566,
"OASv3-yaml": 51239111,
"OCaml": 255,
"Oberon": 677210597,
"ObjDump": 256,
"Object Data Instance Notation": 985227236,
"ObjectScript": 202735509,
"Objective-C": 257,
"Objective-C++": 258,
"Objective-J": 259,
"Odin": 889244082,
"Omgrofl": 260,
"Opa": 261,
"Opal": 262,
"Open Policy Agent": 840483232,
"OpenAPI Specification v2": 848295328,
"OpenAPI Specification v3": 557959099,
"OpenCL": 263,
"OpenEdge ABL": 264,
"OpenQASM": 153739399,
"OpenRC runscript": 265,
"OpenSCAD": 266,
"OpenStep Property List": 598917541,
"OpenType Feature File": 374317347,
"Option List": 723589315,
"Org": 267,
"Ox": 268,
"Oxygene": 269,
"Oz": 270,
"P4": 348895984,
"PDDL": 736235603,
"PEG.js": 81442128,
"PHP": 272,
"PLSQL": 273,
"PLpgSQL": 274,
"POV-Ray SDL": 275,
"Pact": 756774415,
"Pan": 276,
"Papyrus": 277,
"Parrot": 278,
"Parrot Assembly": 279,
"Parrot Internal Representation": 280,
"Pascal": 281,
"Pawn": 271,
"Pep8": 840372442,
"Perl": 282,
"Pic": 425,
"Pickle": 284,
"PicoLisp": 285,
"PigLatin": 286,
"Pike": 287,
"Pip Requirements": 684385621,
"Pkl": 288822799,
"PlantUML": 833504686,
"Pod": 288,
"Pod 6": 155357471,
"PogoScript": 289,
"Polar": 839112914,
"Pony": 290,
"Portugol": 832391833,
"PostCSS": 262764437,
"PostScript": 291,
"PowerBuilder": 292,
"PowerShell": 293,
"Praat": 106029007,
"Prisma": 499933428,
"Processing": 294,
"Procfile": 305313959,
"Proguard": 716513858,
"Prolog": 295,
"Promela": 441858312,
"Propeller Spin": 296,
"Protocol Buffer": 297,
"Protocol Buffer Text Format": 436568854,
"Public Key": 298,
"Pug": 179,
"Puppet": 299,
"Pure Data": 300,
"PureBasic": 301,
"PureScript": 302,
"Pyret": 252961827,
"Python": 303,
"Python console": 428,
"Python traceback": 304,
"Q#": 697448245,
"QML": 305,
"QMake": 306,
"Qt Script": 558193693,
"Quake": 375265331,
"R": 307,
"RAML": 308,
"RBS": 899227493,
"RDoc": 309,
"REALbasic": 310,
"REXX": 311,
"RMarkdown": 313,
"RON": 587855233,
"RPC": 1031374237,
"RPGLE": 609977990,
"RPM Spec": 314,
"RUNOFF": 315,
"Racket": 316,
"Ragel": 317,
"Raku": 283,
"Rascal": 173616037,
"Raw token data": 318,
"ReScript": 501875647,
"Readline Config": 538732839,
"Reason": 869538413,
"ReasonLIGO": 319002153,
"Rebol": 319,
"Record Jar": 865765202,
"Red": 320,
"Redcode": 321,
"Redirect Rules": 1020148948,
"Regular Expression": 363378884,
"Ren'Py": 322,
"RenderScript": 323,
"Rez": 498022874,
"Rich Text Format": 51601661,
"Ring": 431,
"Riot": 878396783,
"RobotFramework": 324,
"Roc": 440182480,
"Roff": 141,
"Roff Manpage": 612669833,
"Rouge": 325,
"RouterOS Script": 592853203,
"Ruby": 326,
"Rust": 327,
"SAS": 328,
"SCSS": 329,
"SELinux Policy": 880010326,
"SMT": 330,
"SPARQL": 331,
"SQF": 332,
"SQL": 333,
"SQLPL": 334,
"SRecode Template": 335,
"SSH Config": 554920715,
"STAR": 424510560,
"STL": 455361735,
"STON": 336,
"SVG": 337,
"SWIG": 1066250075,
"Sage": 338,
"SaltStack": 339,
"Sass": 340,
"Scala": 341,
"Scaml": 342,
"Scenic": 619814037,
"Scheme": 343,
"Scilab": 344,
"Self": 345,
"ShaderLab": 664257356,
"Shell": 346,
"ShellCheck Config": 687511714,
"ShellSession": 347,
"Shen": 348,
"Sieve": 208976687,
"Simple File Verification": 735623761,
"Singularity": 987024632,
"Slash": 349,
"Slice": 894641667,
"Slim": 350,
"Slint": 119900149,
"SmPL": 164123055,
"Smali": 351,
"Smalltalk": 352,
"Smarty": 353,
"Smithy": 1027892786,
"Snakemake": 151241392,
"Solidity": 237469032,
"Soong": 222900098,
"SourcePawn": 354,
"Spline Font Database": 767169629,
"Squirrel": 355,
"Stan": 356,
"Standard ML": 357,
"Starlark": 960266174,
"Stata": 358,
"StringTemplate": 89855901,
"Stylus": 359,
"SubRip Text": 360,
"SugarSS": 826404698,
"SuperCollider": 361,
"Svelte": 928734530,
"Sway": 271471144,
"Sweave": 558779190,
"Swift": 362,
"SystemVerilog": 363,
"TI Program": 422,
"TL-Verilog": 118656070,
"TLA": 364,
"TOML": 365,
"TSQL": 918334941,
"TSV": 1035892117,
"TSX": 94901924,
"TXL": 366,
"Talon": 959889508,
"Tcl": 367,
"Tcsh": 368,
"TeX": 369,
"Tea": 370,
"Terra": 371,
"Terraform Template": 856832701,
"Texinfo": 988020015,
"Text": 372,
"TextGrid": 965696054,
"TextMate Properties": 981795023,
"Textile": 373,
"Thrift": 374,
"Toit": 356554395,
"Turing": 375,
"Turtle": 376,
"Twig": 377,
"Type Language": 632765617,
"TypeScript": 378,
"Typst": 704730682,
"Unified Parallel C": 379,
"Unity3D Asset": 380,
"Unix Assembly": 120,
"Uno": 381,
"UnrealScript": 382,
"UrWeb": 383,
"V": 603371597,
"VBA": 399230729,
"VBScript": 408016005,
"VCL": 384,
"VHDL": 385,
"Vala": 386,
"Valve Data Format": 544060961,
"Velocity Template Language": 292377326,
"Verilog": 387,
"Vim Help File": 508563686,
"Vim Script": 388,
"Vim Snippet": 81265970,
"Visual Basic .NET": 389,
"Visual Basic 6.0": 679594952,
"Volt": 390,
"Vue": 391,
"Vyper": 1055641948,
"WDL": 374521672,
"WGSL": 836605993,
"Wavefront Material": 392,
"Wavefront Object": 393,
"Web Ontology Language": 394,
"WebAssembly": 956556503,
"WebAssembly Interface Type": 134534086,
"WebIDL": 395,
"WebVTT": 658679714,
"Wget Config": 668457123,
"Whiley": 888779559,
"Wikitext": 228,
"Win32 Message File": 950967261,
"Windows Registry Entries": 969674868,
"Witcher Script": 686821385,
"Wollok": 632745969,
"World of Warcraft Addon Data": 396,
"Wren": 713580619,
"X BitMap": 782911107,
"X Font Directory Index": 208700028,
"X PixMap": 781846279,
"X10": 397,
"XC": 398,
"XCompose": 225167241,
"XML": 399,
"XML Property List": 75622871,
"XPages": 400,
"XProc": 401,
"XQuery": 402,
"XS": 403,
"XSLT": 404,
"Xojo": 405,
"Xonsh": 614078284,
"Xtend": 406,
"YAML": 407,
"YANG": 408,
"YARA": 805122868,
"YASnippet": 378760102,
"Yacc": 409,
"Yul": 237469033,
"ZAP": 952972794,
"ZIL": 973483626,
"Zeek": 40,
"ZenScript": 494938890,
"Zephir": 410,
"Zig": 646424281,
"Zimpl": 411,
"cURL Config": 992375436,
"crontab": 705203557,
"desktop": 412,
"dircolors": 691605112,
"eC": 413,
"edn": 414,
"fish": 415,
"hoon": 560883276,
"jq": 905371884,
"kvlang": 970675279,
"mIRC Script": 517654727,
"mcfunction": 462488745,
"mupad": 416,
"nanorc": 775996197,
"nesC": 417,
"ooc": 418,
"q": 970539067,
"reStructuredText": 419,
"robots.txt": 674736065,
"sed": 847830017,
"templ": 795579337,
"wisp": 420,
"xBase": 421,
}

160
data/interpreter.go Normal file
View File

@ -0,0 +1,160 @@
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 42fd3c2514375fc6ad281552368edd3fb9f6ee7f
package data
var LanguagesByInterpreter = map[string][]string{
"M2": {"Macaulay2"},
"RouterOS": {"RouterOS Script"},
"Rscript": {"R"},
"aidl": {"AIDL"},
"apl": {"APL"},
"aplx": {"APL"},
"ash": {"Shell"},
"asy": {"Asymptote"},
"awk": {"Awk"},
"bash": {"Shell"},
"bb": {"Clojure"},
"bigloo": {"Scheme"},
"boogie": {"Boogie"},
"boolector": {"SMT"},
"ccl": {"Common Lisp"},
"chakra": {"JavaScript"},
"chicken": {"Scheme"},
"clisp": {"Common Lisp"},
"coffee": {"CoffeeScript"},
"cperl": {"Perl"},
"crystal": {"Crystal"},
"csh": {"Tcsh"},
"csi": {"Scheme"},
"cvc4": {"SMT"},
"cwl-runner": {"Common Workflow Language"},
"d8": {"JavaScript"},
"dafny": {"Dafny"},
"dart": {"Dart"},
"dash": {"Shell"},
"deno": {"TypeScript"},
"dtrace": {"DTrace"},
"dyalog": {"APL"},
"ecl": {"Common Lisp"},
"elixir": {"Elixir"},
"elvish": {"Elvish"},
"escript": {"Erlang"},
"eui": {"Euphoria"},
"euiw": {"Euphoria"},
"fennel": {"Fennel"},
"fish": {"fish"},
"gawk": {"Awk"},
"gerbv": {"Gerber Image"},
"gerbview": {"Gerber Image"},
"gjs": {"JavaScript"},
"gn": {"GN"},
"gnuplot": {"Gnuplot"},
"gojq": {"jq"},
"gosh": {"Scheme"},
"groovy": {"Groovy"},
"gsed": {"sed"},
"guile": {"Scheme"},
"hy": {"Hy"},
"instantfpc": {"Pascal"},
"io": {"Io"},
"ioke": {"Ioke"},
"janet": {"Janet"},
"jaq": {"jq"},
"jconsole": {"J"},
"jolie": {"Jolie"},
"jq": {"jq"},
"jqjq": {"jq"},
"jqq": {"jq"},
"jruby": {"Ruby"},
"js": {"JavaScript"},
"julia": {"Julia"},
"ksh": {"Shell"},
"lisp": {"Common Lisp"},
"lsl": {"LSL"},
"lua": {"Lua", "Terra"},
"luau": {"Luau"},
"macruby": {"Ruby"},
"make": {"Makefile"},
"makeinfo": {"Texinfo"},
"mathsat5": {"SMT"},
"mawk": {"Awk"},
"minised": {"sed"},
"mksh": {"Shell"},
"mmi": {"Mercury"},
"moon": {"MoonScript"},
"nawk": {"Awk"},
"newlisp": {"NewLisp"},
"nextflow": {"Nextflow"},
"node": {"JavaScript"},
"nodejs": {"JavaScript"},
"nu": {"Nushell"},
"nush": {"Nu"},
"ocaml": {"OCaml", "ReScript"},
"ocamlrun": {"OCaml"},
"ocamlscript": {"OCaml"},
"openrc-run": {"OpenRC runscript"},
"opensmt": {"SMT"},
"osascript": {"AppleScript"},
"parrot": {"Parrot Assembly", "Parrot Internal Representation"},
"pdksh": {"Shell"},
"perl": {"Perl", "Pod"},
"perl6": {"Pod 6", "Raku"},
"php": {"PHP"},
"picolisp": {"PicoLisp"},
"pike": {"Pike"},
"pil": {"PicoLisp"},
"pkl": {"Pkl"},
"pwsh": {"PowerShell"},
"py": {"Python"},
"pypy": {"Python"},
"pypy3": {"Python"},
"python": {"Python"},
"python2": {"Python"},
"python3": {"Python"},
"qjs": {"JavaScript"},
"qmake": {"QMake"},
"query-json": {"jq"},
"r6rs": {"Scheme"},
"racket": {"Racket"},
"rake": {"Ruby"},
"raku": {"Raku"},
"rakudo": {"Raku"},
"rbx": {"Ruby"},
"rc": {"Shell"},
"regina": {"REXX"},
"rexx": {"REXX"},
"rhino": {"JavaScript"},
"ruby": {"Ruby"},
"rune": {"E"},
"runghc": {"Haskell"},
"runhaskell": {"Haskell"},
"runhugs": {"Haskell"},
"rust-script": {"Rust"},
"sbcl": {"Common Lisp"},
"scala": {"Scala"},
"scenic": {"Scenic"},
"scheme": {"Scheme"},
"sclang": {"SuperCollider"},
"scsynth": {"SuperCollider"},
"sed": {"sed"},
"sh": {"Shell"},
"smt-rat": {"SMT"},
"smtinterpol": {"SMT"},
"ssed": {"sed"},
"stp": {"SMT"},
"swipl": {"Prolog"},
"tcc": {"C"},
"tclsh": {"Tcl"},
"tcsh": {"Tcsh"},
"ts-node": {"TypeScript"},
"tsx": {"TypeScript"},
"v8": {"JavaScript"},
"v8-shell": {"JavaScript"},
"verit": {"SMT"},
"wish": {"Tcl"},
"yap": {"Prolog"},
"yices2": {"SMT"},
"z3": {"SMT"},
"zsh": {"Shell"},
}

16034
data/languageInfo.go Normal file
View File

File diff suppressed because it is too large Load Diff

282
data/mimeType.go Normal file
View File

@ -0,0 +1,282 @@
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 42fd3c2514375fc6ad281552368edd3fb9f6ee7f
package data
var LanguagesMime = map[string]string{
"AGS Script": "text/x-c++src",
"APL": "text/apl",
"ASN.1": "text/x-ttcn-asn",
"ASP.NET": "application/x-aspx",
"Alpine Abuild": "text/x-sh",
"AngelScript": "text/x-c++src",
"Ant Build System": "application/xml",
"Apex": "text/x-java",
"Astro": "text/jsx",
"Asymptote": "text/x-kotlin",
"Beef": "text/x-csharp",
"BibTeX": "text/x-stex",
"Bikeshed": "text/html",
"Bluespec": "text/x-systemverilog",
"Bluespec BH": "text/x-haskell",
"Brainfuck": "text/x-brainfuck",
"C": "text/x-csrc",
"C#": "text/x-csharp",
"C++": "text/x-c++src",
"C2hs Haskell": "text/x-haskell",
"CMake": "text/x-cmake",
"COBOL": "text/x-cobol",
"COLLADA": "text/xml",
"CSON": "text/x-coffeescript",
"CSS": "text/css",
"Cabal Config": "text/x-haskell",
"CameLIGO": "text/x-ocaml",
"ChucK": "text/x-java",
"Clojure": "text/x-clojure",
"Closure Templates": "text/x-soy",
"Cloud Firestore Security Rules": "text/css",
"CoffeeScript": "text/x-coffeescript",
"Common Lisp": "text/x-common-lisp",
"Common Workflow Language": "text/x-yaml",
"Component Pascal": "text/x-pascal",
"Crystal": "text/x-crystal",
"Cuda": "text/x-c++src",
"Cycript": "text/javascript",
"Cython": "text/x-cython",
"D": "text/x-d",
"DTrace": "text/x-csrc",
"Dart": "application/dart",
"DenizenScript": "text/x-yaml",
"Dhall": "text/x-haskell",
"Diff": "text/x-diff",
"Dockerfile": "text/x-dockerfile",
"Dylan": "text/x-dylan",
"E-mail": "application/mbox",
"EBNF": "text/x-ebnf",
"ECL": "text/x-ecl",
"EQ": "text/x-csharp",
"Eagle": "text/xml",
"Easybuild": "text/x-python",
"Ecere Projects": "application/json",
"Ecmarkup": "text/html",
"EditorConfig": "text/x-properties",
"Edje Data Collection": "text/x-c++src",
"Eiffel": "text/x-eiffel",
"Elm": "text/x-elm",
"Emacs Lisp": "text/x-common-lisp",
"EmberScript": "text/x-coffeescript",
"Erlang": "text/x-erlang",
"F#": "text/x-fsharp",
"Factor": "text/x-factor",
"Forth": "text/x-forth",
"Fortran": "text/x-fortran",
"Fortran Free Form": "text/x-fortran",
"FreeBasic": "text/x-vb",
"GCC Machine Description": "text/x-common-lisp",
"GN": "text/x-python",
"GSC": "text/x-csrc",
"Game Maker Language": "text/x-c++src",
"Genshi": "text/xml",
"Gentoo Ebuild": "text/x-sh",
"Gentoo Eclass": "text/x-sh",
"Git Attributes": "text/x-sh",
"Git Config": "text/x-properties",
"Glyph": "text/x-tcl",
"Go": "text/x-go",
"Grammatical Framework": "text/x-haskell",
"Groovy": "text/x-groovy",
"Groovy Server Pages": "application/x-jsp",
"HCL": "text/x-ruby",
"HTML": "text/html",
"HTML+ECR": "text/html",
"HTML+EEX": "text/html",
"HTML+ERB": "application/x-erb",
"HTML+PHP": "application/x-httpd-php",
"HTML+Razor": "text/html",
"HTTP": "message/http",
"Hack": "application/x-httpd-php",
"Haml": "text/x-haml",
"Haskell": "text/x-haskell",
"Haxe": "text/x-haxe",
"HolyC": "text/x-csrc",
"IDL": "text/x-idl",
"INI": "text/x-properties",
"IRC log": "text/mirc",
"Ignore List": "text/x-sh",
"JSON": "application/json",
"JSON with Comments": "text/javascript",
"JSON5": "application/json",
"JSONLD": "application/json",
"JSONiq": "application/json",
"Janet": "text/x-scheme",
"Java": "text/x-java",
"Java Properties": "text/x-properties",
"Java Server Pages": "application/x-jsp",
"JavaScript": "text/javascript",
"JavaScript+ERB": "application/javascript",
"Jest Snapshot": "application/javascript",
"JetBrains MPS": "text/xml",
"Jinja": "text/x-django",
"Julia": "text/x-julia",
"Jupyter Notebook": "application/json",
"Kaitai Struct": "text/x-yaml",
"KiCad Layout": "text/x-common-lisp",
"Kit": "text/html",
"Kotlin": "text/x-kotlin",
"LFE": "text/x-common-lisp",
"LTspice Symbol": "text/x-spreadsheet",
"LabVIEW": "text/xml",
"Lark": "text/x-ebnf",
"Latte": "text/x-smarty",
"Less": "text/css",
"LigoLANG": "text/x-pascal",
"Literate Haskell": "text/x-literate-haskell",
"LiveScript": "text/x-livescript",
"LookML": "text/x-yaml",
"Lua": "text/x-lua",
"Luau": "text/x-lua",
"M": "text/x-mumps",
"MATLAB": "text/x-octave",
"MDX": "text/x-gfm",
"MTML": "text/html",
"MUF": "text/x-forth",
"Makefile": "text/x-cmake",
"Markdown": "text/x-gfm",
"Marko": "text/html",
"Mathematica": "text/x-mathematica",
"Maven POM": "text/xml",
"Max": "application/json",
"Metal": "text/x-c++src",
"MiniYAML": "text/x-yaml",
"Mirah": "text/x-ruby",
"Modelica": "text/x-modelica",
"Mojo": "text/x-python",
"Monkey C": "text/x-csrc",
"Mustache": "text/x-smarty",
"NSIS": "text/x-nsis",
"NWScript": "text/x-csrc",
"NetLogo": "text/x-common-lisp",
"NewLisp": "text/x-common-lisp",
"Nginx": "text/x-nginx-conf",
"Nu": "text/x-scheme",
"NumPy": "text/x-python",
"Nushell": "text/x-sh",
"OASv2-json": "application/json",
"OASv2-yaml": "text/x-yaml",
"OASv3-json": "application/json",
"OASv3-yaml": "text/x-yaml",
"OCaml": "text/x-ocaml",
"Objective-C": "text/x-objectivec",
"Objective-C++": "text/x-objectivec",
"OpenCL": "text/x-csrc",
"OpenRC runscript": "text/x-sh",
"Option List": "text/x-sh",
"Oz": "text/x-oz",
"PEG.js": "text/javascript",
"PHP": "application/x-httpd-php",
"PLSQL": "text/x-plsql",
"PLpgSQL": "text/x-sql",
"Pascal": "text/x-pascal",
"Perl": "text/x-perl",
"Pic": "text/troff",
"Pod": "text/x-perl",
"PowerShell": "application/x-powershell",
"Protocol Buffer": "text/x-protobuf",
"Public Key": "application/pgp",
"Pug": "text/x-pug",
"Puppet": "text/x-puppet",
"PureScript": "text/x-haskell",
"Python": "text/x-python",
"Qt Script": "text/javascript",
"R": "text/x-rsrc",
"RAML": "text/x-yaml",
"RBS": "text/x-ruby",
"RMarkdown": "text/x-gfm",
"RPM Spec": "text/x-rpm-spec",
"Raku": "text/x-perl",
"ReScript": "text/x-rustsrc",
"Reason": "text/x-rustsrc",
"ReasonLIGO": "text/x-rustsrc",
"Record Jar": "text/x-properties",
"Roff": "text/troff",
"Roff Manpage": "text/troff",
"Rouge": "text/x-clojure",
"Ruby": "text/x-ruby",
"Rust": "text/x-rustsrc",
"SAS": "text/x-sas",
"SCSS": "text/x-scss",
"SPARQL": "application/sparql-query",
"SQL": "text/x-sql",
"SQLPL": "text/x-sql",
"SRecode Template": "text/x-common-lisp",
"SVG": "text/xml",
"SWIG": "text/x-c++src",
"Sage": "text/x-python",
"SaltStack": "text/x-yaml",
"Sass": "text/x-sass",
"Scala": "text/x-scala",
"Scheme": "text/x-scheme",
"Shell": "text/x-sh",
"ShellCheck Config": "text/x-properties",
"ShellSession": "text/x-sh",
"Sieve": "application/sieve",
"Simple File Verification": "text/x-properties",
"Slim": "text/x-slim",
"Smalltalk": "text/x-stsrc",
"Smarty": "text/x-smarty",
"Smithy": "text/x-csrc",
"Snakemake": "text/x-python",
"Squirrel": "text/x-c++src",
"Standard ML": "text/x-ocaml",
"Starlark": "text/x-python",
"StringTemplate": "text/html",
"Svelte": "text/html",
"Sway": "text/x-rustsrc",
"Swift": "text/x-swift",
"SystemVerilog": "text/x-systemverilog",
"TOML": "text/x-toml",
"TSX": "text/jsx",
"Tcl": "text/x-tcl",
"Tcsh": "text/x-sh",
"TeX": "text/x-stex",
"Terra": "text/x-lua",
"Terraform Template": "text/x-ruby",
"TextMate Properties": "text/x-properties",
"Textile": "text/x-textile",
"Turtle": "text/turtle",
"Twig": "text/x-twig",
"TypeScript": "application/typescript",
"Unified Parallel C": "text/x-csrc",
"Unity3D Asset": "text/x-yaml",
"Uno": "text/x-csharp",
"UnrealScript": "text/x-java",
"V": "text/x-go",
"VBA": "text/x-vb",
"VBScript": "text/vbscript",
"VHDL": "text/x-vhdl",
"Velocity Template Language": "text/velocity",
"Verilog": "text/x-verilog",
"Visual Basic .NET": "text/x-vb",
"Visual Basic 6.0": "text/x-vb",
"Volt": "text/x-d",
"WebAssembly": "text/x-common-lisp",
"WebAssembly Interface Type": "text/x-webidl",
"WebIDL": "text/x-webidl",
"Win32 Message File": "text/x-properties",
"Windows Registry Entries": "text/x-properties",
"X BitMap": "text/x-csrc",
"X PixMap": "text/x-csrc",
"XC": "text/x-csrc",
"XML": "text/xml",
"XML Property List": "text/xml",
"XPages": "text/xml",
"XProc": "text/xml",
"XQuery": "application/xquery",
"XS": "text/x-csrc",
"XSLT": "text/xml",
"Xonsh": "text/x-python",
"YAML": "text/x-yaml",
"edn": "text/x-clojure",
"reStructuredText": "text/x-rst",
"wisp": "text/x-clojure",
}

133
data/rule/rule.go Normal file
View File

@ -0,0 +1,133 @@
// Package rule contains rule-based heuristic implementations.
// It is used in the generated code in content.go for disambiguation of languages
// with colliding extensions, based on regexps from Linguist data.
package rule
import "github.com/go-enry/go-enry/v2/regex"
// Matcher checks if the data matches (number of) pattern(s).
// Every heuristic rule below implements this interface.
// A regexp.Regexp satisfies this interface and can be used instead.
type Matcher interface {
Match(data []byte) bool
}
// Heuristic consist of (a number of) rules where each, if matches,
// identifies content as belonging to a programming language(s).
type Heuristic interface {
Matcher
Languages() []string
}
// languages base struct with all the languages that a Matcher identifies.
type languages struct {
langs []string
}
// Languages returns all languages, identified by this Matcher.
func (l languages) Languages() []string {
return l.langs
}
// MatchingLanguages is a helper to create new languages.
func MatchingLanguages(langs ...string) languages {
return languages{langs}
}
func noLanguages() languages {
return MatchingLanguages([]string{}...)
}
// Implements a Heuristic.
type or struct {
languages
pattern Matcher
}
// Or rule matches, if a single matching pattern exists.
// It receives only one pattern as it relies on optimization that
// represtes union with | inside a single regexp during code generation.
func Or(l languages, p Matcher) Heuristic {
//FIXME(bzz): this will not be the case as only some of the patterns may
// be non-RE2 => we shouldn't collate them not to loose the (accuracty of) whole rule
return or{l, p}
}
// Match implements rule.Matcher.
func (r or) Match(data []byte) bool {
if runOnRE2AndRegexNotAccepted(r.pattern) {
return false
}
return r.pattern.Match(data)
}
// Implements a Heuristic.
type and struct {
languages
patterns []Matcher
}
// And rule matches, if each of the patterns does match.
func And(l languages, m ...Matcher) Heuristic {
return and{l, m}
}
// Match implements data.Matcher.
func (r and) Match(data []byte) bool {
for _, p := range r.patterns {
if runOnRE2AndRegexNotAccepted(p) {
continue
}
if !p.Match(data) {
return false
}
}
return true
}
// Implements a Heuristic.
type not struct {
languages
Patterns []Matcher
}
// Not rule matches if none of the patterns match.
func Not(l languages, r ...Matcher) Heuristic {
return not{l, r}
}
// Match implements data.Matcher.
func (r not) Match(data []byte) bool {
for _, p := range r.Patterns {
if runOnRE2AndRegexNotAccepted(p) {
continue
}
if p.Match(data) {
return false
}
}
return true
}
// Implements a Heuristic.
type always struct {
languages
}
// Always rule always matches. Often is used as a default fallback.
func Always(l languages) Heuristic {
return always{l}
}
// Match implements Matcher.
func (r always) Match(data []byte) bool {
return true
}
// Checks if a regex syntax isn't accepted by RE2 engine.
// It's nil by construction from regex.MustCompileRuby but
// is used here as a Matcher interface wich itself is non-nil.
func runOnRE2AndRegexNotAccepted(re Matcher) bool {
v, ok := re.(regex.EnryRegexp)
return ok && v == nil
}

71
data/rule/rule_test.go Normal file
View File

@ -0,0 +1,71 @@
package rule
import (
"testing"
"github.com/go-enry/go-enry/v2/regex"
"github.com/stretchr/testify/assert"
)
const lang = "ActionScript"
type fixture struct {
name string
rule Heuristic
numLangs int
match string
noMatch string
}
var specificFixtures = map[string][]fixture{
"": { // cases that don't vary between the engines
{"Always", Always(MatchingLanguages(lang)), 1, "a", ""},
{"Not", Not(MatchingLanguages(lang), regex.MustCompile(`a`)), 1, "b", "a"},
{"And", And(MatchingLanguages(lang), regex.MustCompile(`a`), regex.MustCompile(`b`)), 1, "ab", "a"},
{"Or", Or(MatchingLanguages(lang), regex.MustCompile(`a|b`)), 1, "ab", "c"},
// the results of these depend on the regex engine
// {"NilOr", Or(noLanguages(), regex.MustCompileRuby(``)), 0, "", "a"},
// {"NilNot", Not(noLanguages(), regex.MustCompileRuby(`a`)), 0, "", "a"},
},
regex.RE2: {
{"NilAnd", And(noLanguages(), regex.MustCompileRuby(`a`), regex.MustCompile(`b`)), 0, "b", "a"},
{"NilNot", Not(noLanguages(), regex.MustCompileRuby(`a`), regex.MustCompile(`b`)), 0, "c", "b"},
},
regex.Oniguruma: {
{"NilAnd", And(noLanguages(), regex.MustCompileRuby(`a`), regex.MustCompile(`b`)), 0, "ab", "c"},
{"NilNot", Not(noLanguages(), regex.MustCompileRuby(`a`), regex.MustCompile(`b`)), 0, "c", "a"},
{"NilOr", Or(noLanguages(), regex.MustCompileRuby(`a`) /*, regexp.MustCompile(`b`)*/), 0, "a", "b"},
},
}
func testRulesForEngine(t *testing.T, engine string) {
if engine != "" && regex.Name != engine {
return
}
for _, f := range specificFixtures[engine] {
t.Run(engine+f.name, func(t *testing.T) {
check(t, f)
})
}
}
func TestRules(t *testing.T) {
//TODO(bzz): can all be run in parallel
testRulesForEngine(t, "")
testRulesForEngine(t, regex.RE2)
testRulesForEngine(t, regex.Oniguruma)
}
func check(t *testing.T, f fixture) {
assert.NotNil(t, f.rule)
assert.NotNil(t, f.rule.Languages())
assert.Equal(t, f.numLangs, len(f.rule.Languages()))
if f.match != "" {
assert.Truef(t, f.rule.Match([]byte(f.match)),
"'%s' is expected to .Match() by rule %s%v", f.match, f.name, f.rule)
}
if f.noMatch != "" {
assert.Falsef(t, f.rule.Match([]byte(f.noMatch)),
"'%s' is expected NOT to .Match() by rule %s%v", f.noMatch, f.name, f.rule)
}
}

17
data/test.go Normal file
View File

@ -0,0 +1,17 @@
package data
import "github.com/go-enry/go-enry/v2/regex"
// TestMatchers is hand made collection of regexp used by the function `enry.IsTest`
// to identify test files in different languages.
var TestMatchers = []regex.EnryRegexp{
regex.MustCompile(`(^|/)tests/.*Test\.php$`),
regex.MustCompile(`(^|/)test/.*Test(s?)\.java$`),
regex.MustCompile(`(^|/)test(/|/.*/)Test.*\.java$`),
regex.MustCompile(`(^|/)test/.*(Test(s?)|Spec(s?))\.scala$`),
regex.MustCompile(`(^|/)test_.*\.py$`),
regex.MustCompile(`(^|/).*_test\.go$`),
regex.MustCompile(`(^|/).*_(test|spec)\.rb$`),
regex.MustCompile(`(^|/).*Test(s?)\.cs$`),
regex.MustCompile(`(^|/).*\.(test|spec)\.(ts|tsx|js)$`),
}

779
data/type.go Normal file
View File

@ -0,0 +1,779 @@
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 42fd3c2514375fc6ad281552368edd3fb9f6ee7f
package data
// Type represent language's type. Either data, programming, markup, prose, or unknown.
type Type int
// Type's values.
const (
TypeUnknown Type = iota
TypeData
TypeProgramming
TypeMarkup
TypeProse
)
func (t Type) String() string {
switch t {
case TypeData:
return "data"
case TypeProgramming:
return "programming"
case TypeMarkup:
return "markup"
case TypeProse:
return "prose"
default:
return "unknown"
}
}
func TypeForString(s string) Type {
switch s {
case "data":
return TypeData
case "programming":
return TypeProgramming
case "markup":
return TypeMarkup
case "prose":
return TypeProse
default:
return TypeUnknown
}
}
var LanguagesType = map[string]int{
"1C Enterprise": 2,
"2-Dimensional Array": 1,
"4D": 2,
"ABAP": 2,
"ABAP CDS": 2,
"ABNF": 1,
"AGS Script": 2,
"AIDL": 2,
"AL": 2,
"AMPL": 2,
"ANTLR": 2,
"API Blueprint": 3,
"APL": 2,
"ASL": 2,
"ASN.1": 1,
"ASP.NET": 2,
"ATS": 2,
"ActionScript": 2,
"Ada": 2,
"Adblock Filter List": 1,
"Adobe Font Metrics": 1,
"Agda": 2,
"Alloy": 2,
"Alpine Abuild": 2,
"Altium Designer": 1,
"AngelScript": 2,
"Ant Build System": 1,
"Antlers": 3,
"ApacheConf": 1,
"Apex": 2,
"Apollo Guidance Computer": 2,
"AppleScript": 2,
"Arc": 2,
"AsciiDoc": 4,
"AspectJ": 2,
"Assembly": 2,
"Astro": 3,
"Asymptote": 2,
"Augeas": 2,
"AutoHotkey": 2,
"AutoIt": 2,
"Avro IDL": 1,
"Awk": 2,
"BASIC": 2,
"BQN": 2,
"Ballerina": 2,
"Batchfile": 2,
"Beef": 2,
"Befunge": 2,
"Berry": 2,
"BibTeX": 3,
"Bicep": 2,
"Bikeshed": 3,
"Bison": 2,
"BitBake": 2,
"Blade": 3,
"BlitzBasic": 2,
"BlitzMax": 2,
"Bluespec": 2,
"Bluespec BH": 2,
"Boo": 2,
"Boogie": 2,
"Brainfuck": 2,
"BrighterScript": 2,
"Brightscript": 2,
"Browserslist": 1,
"C": 2,
"C#": 2,
"C++": 2,
"C-ObjDump": 1,
"C2hs Haskell": 2,
"CAP CDS": 2,
"CIL": 1,
"CLIPS": 2,
"CMake": 2,
"COBOL": 2,
"CODEOWNERS": 1,
"COLLADA": 1,
"CSON": 1,
"CSS": 3,
"CSV": 1,
"CUE": 2,
"CWeb": 2,
"Cabal Config": 1,
"Caddyfile": 1,
"Cadence": 2,
"Cairo": 2,
"CameLIGO": 2,
"Cap'n Proto": 2,
"CartoCSS": 2,
"Ceylon": 2,
"Chapel": 2,
"Charity": 2,
"Checksums": 1,
"ChucK": 2,
"Circom": 2,
"Cirru": 2,
"Clarion": 2,
"Clarity": 2,
"Classic ASP": 2,
"Clean": 2,
"Click": 2,
"Clojure": 2,
"Closure Templates": 3,
"Cloud Firestore Security Rules": 1,
"CoNLL-U": 1,
"CodeQL": 2,
"CoffeeScript": 2,
"ColdFusion": 2,
"ColdFusion CFC": 2,
"Common Lisp": 2,
"Common Workflow Language": 2,
"Component Pascal": 2,
"Cool": 2,
"Coq": 2,
"Cpp-ObjDump": 1,
"Creole": 4,
"Crystal": 2,
"Csound": 2,
"Csound Document": 2,
"Csound Score": 2,
"Cuda": 2,
"Cue Sheet": 1,
"Curry": 2,
"Cycript": 2,
"Cypher": 2,
"Cython": 2,
"D": 2,
"D-ObjDump": 1,
"D2": 3,
"DIGITAL Command Language": 2,
"DM": 2,
"DNS Zone": 1,
"DTrace": 2,
"Dafny": 2,
"Darcs Patch": 1,
"Dart": 2,
"DataWeave": 2,
"Debian Package Control File": 1,
"DenizenScript": 2,
"Dhall": 2,
"Diff": 1,
"DirectX 3D File": 1,
"Dockerfile": 2,
"Dogescript": 2,
"Dotenv": 1,
"Dylan": 2,
"E": 2,
"E-mail": 1,
"EBNF": 1,
"ECL": 2,
"ECLiPSe": 2,
"EJS": 3,
"EQ": 2,
"Eagle": 1,
"Earthly": 2,
"Easybuild": 1,
"Ecere Projects": 1,
"Ecmarkup": 3,
"Edge": 3,
"EdgeQL": 2,
"EditorConfig": 1,
"Edje Data Collection": 1,
"Eiffel": 2,
"Elixir": 2,
"Elm": 2,
"Elvish": 2,
"Elvish Transcript": 2,
"Emacs Lisp": 2,
"EmberScript": 2,
"Erlang": 2,
"Euphoria": 2,
"F#": 2,
"F*": 2,
"FIGlet Font": 1,
"FIRRTL": 2,
"FLUX": 2,
"Factor": 2,
"Fancy": 2,
"Fantom": 2,
"Faust": 2,
"Fennel": 2,
"Filebench WML": 2,
"Filterscript": 2,
"Fluent": 2,
"Formatted": 1,
"Forth": 2,
"Fortran": 2,
"Fortran Free Form": 2,
"FreeBasic": 2,
"FreeMarker": 2,
"Frege": 2,
"Futhark": 2,
"G-code": 2,
"GAML": 2,
"GAMS": 2,
"GAP": 2,
"GCC Machine Description": 2,
"GDB": 2,
"GDScript": 2,
"GEDCOM": 1,
"GLSL": 2,
"GN": 1,
"GSC": 2,
"Game Maker Language": 2,
"Gemfile.lock": 1,
"Gemini": 4,
"Genero 4gl": 2,
"Genero per": 3,
"Genie": 2,
"Genshi": 2,
"Gentoo Ebuild": 2,
"Gentoo Eclass": 2,
"Gerber Image": 1,
"Gettext Catalog": 4,
"Gherkin": 2,
"Git Attributes": 1,
"Git Config": 1,
"Git Revision List": 1,
"Gleam": 2,
"Glimmer JS": 2,
"Glimmer TS": 2,
"Glyph": 2,
"Glyph Bitmap Distribution Format": 1,
"Gnuplot": 2,
"Go": 2,
"Go Checksums": 1,
"Go Module": 1,
"Go Workspace": 1,
"Godot Resource": 1,
"Golo": 2,
"Gosu": 2,
"Grace": 2,
"Gradle": 1,
"Gradle Kotlin DSL": 1,
"Grammatical Framework": 2,
"Graph Modeling Language": 1,
"GraphQL": 1,
"Graphviz (DOT)": 1,
"Groovy": 2,
"Groovy Server Pages": 2,
"HAProxy": 1,
"HCL": 2,
"HLSL": 2,
"HOCON": 1,
"HTML": 3,
"HTML+ECR": 3,
"HTML+EEX": 3,
"HTML+ERB": 3,
"HTML+PHP": 3,
"HTML+Razor": 3,
"HTTP": 1,
"HXML": 1,
"Hack": 2,
"Haml": 3,
"Handlebars": 3,
"Harbour": 2,
"Haskell": 2,
"Haxe": 2,
"HiveQL": 2,
"HolyC": 2,
"Hosts File": 1,
"Hy": 2,
"HyPhy": 2,
"IDL": 2,
"IGOR Pro": 2,
"INI": 1,
"IRC log": 1,
"Idris": 2,
"Ignore List": 1,
"ImageJ Macro": 2,
"Imba": 2,
"Inform 7": 2,
"Ink": 2,
"Inno Setup": 2,
"Io": 2,
"Ioke": 2,
"Isabelle": 2,
"Isabelle ROOT": 2,
"J": 2,
"JAR Manifest": 1,
"JCL": 2,
"JFlex": 2,
"JSON": 1,
"JSON with Comments": 1,
"JSON5": 1,
"JSONLD": 1,
"JSONiq": 2,
"Janet": 2,
"Jasmin": 2,
"Java": 2,
"Java Properties": 1,
"Java Server Pages": 2,
"JavaScript": 2,
"JavaScript+ERB": 2,
"Jest Snapshot": 1,
"JetBrains MPS": 2,
"Jinja": 3,
"Jison": 2,
"Jison Lex": 2,
"Jolie": 2,
"Jsonnet": 2,
"Julia": 2,
"Julia REPL": 2,
"Jupyter Notebook": 3,
"Just": 2,
"KRL": 2,
"Kaitai Struct": 2,
"KakouneScript": 2,
"KerboScript": 2,
"KiCad Layout": 1,
"KiCad Legacy Layout": 1,
"KiCad Schematic": 1,
"Kickstart": 1,
"Kit": 3,
"Kotlin": 2,
"Kusto": 1,
"LFE": 2,
"LLVM": 2,
"LOLCODE": 2,
"LSL": 2,
"LTspice Symbol": 1,
"LabVIEW": 2,
"Lark": 1,
"Lasso": 2,
"Latte": 3,
"Lean": 2,
"Lean 4": 2,
"Less": 3,
"Lex": 2,
"LigoLANG": 2,
"LilyPond": 2,
"Limbo": 2,
"Linker Script": 1,
"Linux Kernel Module": 1,
"Liquid": 3,
"Literate Agda": 2,
"Literate CoffeeScript": 2,
"Literate Haskell": 2,
"LiveScript": 2,
"Logos": 2,
"Logtalk": 2,
"LookML": 2,
"LoomScript": 2,
"Lua": 2,
"Luau": 2,
"M": 2,
"M4": 2,
"M4Sugar": 2,
"MATLAB": 2,
"MAXScript": 2,
"MDX": 3,
"MLIR": 2,
"MQL4": 2,
"MQL5": 2,
"MTML": 3,
"MUF": 2,
"Macaulay2": 2,
"Makefile": 2,
"Mako": 2,
"Markdown": 4,
"Marko": 3,
"Mask": 3,
"Mathematica": 2,
"Maven POM": 1,
"Max": 2,
"Mercury": 2,
"Mermaid": 3,
"Meson": 2,
"Metal": 2,
"Microsoft Developer Studio Project": 1,
"Microsoft Visual Studio Solution": 1,
"MiniD": 2,
"MiniYAML": 1,
"Mint": 2,
"Mirah": 2,
"Modelica": 2,
"Modula-2": 2,
"Modula-3": 2,
"Module Management System": 2,
"Mojo": 2,
"Monkey": 2,
"Monkey C": 2,
"Moocode": 2,
"MoonScript": 2,
"Motoko": 2,
"Motorola 68K Assembly": 2,
"Move": 2,
"Muse": 4,
"Mustache": 3,
"Myghty": 2,
"NASL": 2,
"NCL": 2,
"NEON": 1,
"NL": 1,
"NMODL": 2,
"NPM Config": 1,
"NSIS": 2,
"NWScript": 2,
"Nasal": 2,
"Nearley": 2,
"Nemerle": 2,
"NetLinx": 2,
"NetLinx+ERB": 2,
"NetLogo": 2,
"NewLisp": 2,
"Nextflow": 2,
"Nginx": 1,
"Nim": 2,
"Ninja": 1,
"Nit": 2,
"Nix": 2,
"Nu": 2,
"NumPy": 2,
"Nunjucks": 3,
"Nushell": 2,
"OASv2-json": 1,
"OASv2-yaml": 1,
"OASv3-json": 1,
"OASv3-yaml": 1,
"OCaml": 2,
"Oberon": 2,
"ObjDump": 1,
"Object Data Instance Notation": 1,
"ObjectScript": 2,
"Objective-C": 2,
"Objective-C++": 2,
"Objective-J": 2,
"Odin": 2,
"Omgrofl": 2,
"Opa": 2,
"Opal": 2,
"Open Policy Agent": 2,
"OpenAPI Specification v2": 1,
"OpenAPI Specification v3": 1,
"OpenCL": 2,
"OpenEdge ABL": 2,
"OpenQASM": 2,
"OpenRC runscript": 2,
"OpenSCAD": 2,
"OpenStep Property List": 1,
"OpenType Feature File": 1,
"Option List": 1,
"Org": 4,
"Ox": 2,
"Oxygene": 2,
"Oz": 2,
"P4": 2,
"PDDL": 2,
"PEG.js": 2,
"PHP": 2,
"PLSQL": 2,
"PLpgSQL": 2,
"POV-Ray SDL": 2,
"Pact": 2,
"Pan": 2,
"Papyrus": 2,
"Parrot": 2,
"Parrot Assembly": 2,
"Parrot Internal Representation": 2,
"Pascal": 2,
"Pawn": 2,
"Pep8": 2,
"Perl": 2,
"Pic": 3,
"Pickle": 1,
"PicoLisp": 2,
"PigLatin": 2,
"Pike": 2,
"Pip Requirements": 1,
"Pkl": 2,
"PlantUML": 1,
"Pod": 4,
"Pod 6": 4,
"PogoScript": 2,
"Polar": 2,
"Pony": 2,
"Portugol": 2,
"PostCSS": 3,
"PostScript": 3,
"PowerBuilder": 2,
"PowerShell": 2,
"Praat": 2,
"Prisma": 1,
"Processing": 2,
"Procfile": 2,
"Proguard": 1,
"Prolog": 2,
"Promela": 2,
"Propeller Spin": 2,
"Protocol Buffer": 1,
"Protocol Buffer Text Format": 1,
"Public Key": 1,
"Pug": 3,
"Puppet": 2,
"Pure Data": 1,
"PureBasic": 2,
"PureScript": 2,
"Pyret": 2,
"Python": 2,
"Python console": 2,
"Python traceback": 1,
"Q#": 2,
"QML": 2,
"QMake": 2,
"Qt Script": 2,
"Quake": 2,
"R": 2,
"RAML": 3,
"RBS": 1,
"RDoc": 4,
"REALbasic": 2,
"REXX": 2,
"RMarkdown": 4,
"RON": 1,
"RPC": 2,
"RPGLE": 2,
"RPM Spec": 1,
"RUNOFF": 3,
"Racket": 2,
"Ragel": 2,
"Raku": 2,
"Rascal": 2,
"Raw token data": 1,
"ReScript": 2,
"Readline Config": 1,
"Reason": 2,
"ReasonLIGO": 2,
"Rebol": 2,
"Record Jar": 1,
"Red": 2,
"Redcode": 2,
"Redirect Rules": 1,
"Regular Expression": 1,
"Ren'Py": 2,
"RenderScript": 2,
"Rez": 2,
"Rich Text Format": 3,
"Ring": 2,
"Riot": 3,
"RobotFramework": 2,
"Roc": 2,
"Roff": 3,
"Roff Manpage": 3,
"Rouge": 2,
"RouterOS Script": 2,
"Ruby": 2,
"Rust": 2,
"SAS": 2,
"SCSS": 3,
"SELinux Policy": 1,
"SMT": 2,
"SPARQL": 1,
"SQF": 2,
"SQL": 1,
"SQLPL": 2,
"SRecode Template": 3,
"SSH Config": 1,
"STAR": 1,
"STL": 1,
"STON": 1,
"SVG": 1,
"SWIG": 2,
"Sage": 2,
"SaltStack": 2,
"Sass": 3,
"Scala": 2,
"Scaml": 3,
"Scenic": 2,
"Scheme": 2,
"Scilab": 2,
"Self": 2,
"ShaderLab": 2,
"Shell": 2,
"ShellCheck Config": 1,
"ShellSession": 2,
"Shen": 2,
"Sieve": 2,
"Simple File Verification": 1,
"Singularity": 2,
"Slash": 2,
"Slice": 2,
"Slim": 3,
"Slint": 3,
"SmPL": 2,
"Smali": 2,
"Smalltalk": 2,
"Smarty": 2,
"Smithy": 2,
"Snakemake": 2,
"Solidity": 2,
"Soong": 1,
"SourcePawn": 2,
"Spline Font Database": 1,
"Squirrel": 2,
"Stan": 2,
"Standard ML": 2,
"Starlark": 2,
"Stata": 2,
"StringTemplate": 3,
"Stylus": 3,
"SubRip Text": 1,
"SugarSS": 3,
"SuperCollider": 2,
"Svelte": 3,
"Sway": 2,
"Sweave": 4,
"Swift": 2,
"SystemVerilog": 2,
"TI Program": 2,
"TL-Verilog": 2,
"TLA": 2,
"TOML": 1,
"TSQL": 2,
"TSV": 1,
"TSX": 2,
"TXL": 2,
"Talon": 2,
"Tcl": 2,
"Tcsh": 2,
"TeX": 3,
"Tea": 3,
"Terra": 2,
"Terraform Template": 3,
"Texinfo": 4,
"Text": 4,
"TextGrid": 1,
"TextMate Properties": 1,
"Textile": 4,
"Thrift": 2,
"Toit": 2,
"Turing": 2,
"Turtle": 1,
"Twig": 3,
"Type Language": 1,
"TypeScript": 2,
"Typst": 2,
"Unified Parallel C": 2,
"Unity3D Asset": 1,
"Unix Assembly": 2,
"Uno": 2,
"UnrealScript": 2,
"UrWeb": 2,
"V": 2,
"VBA": 2,
"VBScript": 2,
"VCL": 2,
"VHDL": 2,
"Vala": 2,
"Valve Data Format": 1,
"Velocity Template Language": 3,
"Verilog": 2,
"Vim Help File": 4,
"Vim Script": 2,
"Vim Snippet": 3,
"Visual Basic .NET": 2,
"Visual Basic 6.0": 2,
"Volt": 2,
"Vue": 3,
"Vyper": 2,
"WDL": 2,
"WGSL": 2,
"Wavefront Material": 1,
"Wavefront Object": 1,
"Web Ontology Language": 1,
"WebAssembly": 2,
"WebAssembly Interface Type": 1,
"WebIDL": 2,
"WebVTT": 1,
"Wget Config": 1,
"Whiley": 2,
"Wikitext": 4,
"Win32 Message File": 1,
"Windows Registry Entries": 1,
"Witcher Script": 2,
"Wollok": 2,
"World of Warcraft Addon Data": 1,
"Wren": 2,
"X BitMap": 1,
"X Font Directory Index": 1,
"X PixMap": 1,
"X10": 2,
"XC": 2,
"XCompose": 1,
"XML": 1,
"XML Property List": 1,
"XPages": 1,
"XProc": 2,
"XQuery": 2,
"XS": 2,
"XSLT": 2,
"Xojo": 2,
"Xonsh": 2,
"Xtend": 2,
"YAML": 1,
"YANG": 1,
"YARA": 2,
"YASnippet": 3,
"Yacc": 2,
"Yul": 2,
"ZAP": 2,
"ZIL": 2,
"Zeek": 2,
"ZenScript": 2,
"Zephir": 2,
"Zig": 2,
"Zimpl": 2,
"cURL Config": 1,
"crontab": 1,
"desktop": 1,
"dircolors": 1,
"eC": 2,
"edn": 1,
"fish": 2,
"hoon": 2,
"jq": 2,
"kvlang": 3,
"mIRC Script": 2,
"mcfunction": 2,
"mupad": 2,
"nanorc": 1,
"nesC": 2,
"ooc": 2,
"q": 2,
"reStructuredText": 4,
"robots.txt": 1,
"sed": 2,
"templ": 3,
"wisp": 2,
"xBase": 2,
}

179
data/vendor.go Normal file
View File

@ -0,0 +1,179 @@
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 42fd3c2514375fc6ad281552368edd3fb9f6ee7f
package data
import "github.com/go-enry/go-enry/v2/regex"
var VendorMatchers = []regex.EnryRegexp{
regex.MustCompile(`(^|/)cache/`),
regex.MustCompile(`^[Dd]ependencies/`),
regex.MustCompile(`(^|/)dist/`),
regex.MustCompile(`^deps/`),
regex.MustCompile(`(^|/)configure$`),
regex.MustCompile(`(^|/)config\.guess$`),
regex.MustCompile(`(^|/)config\.sub$`),
regex.MustCompile(`(^|/)aclocal\.m4`),
regex.MustCompile(`(^|/)libtool\.m4`),
regex.MustCompile(`(^|/)ltoptions\.m4`),
regex.MustCompile(`(^|/)ltsugar\.m4`),
regex.MustCompile(`(^|/)ltversion\.m4`),
regex.MustCompile(`(^|/)lt~obsolete\.m4`),
regex.MustCompile(`(^|/)dotnet-install\.(ps1|sh)$`),
regex.MustCompile(`(^|/)cpplint\.py`),
regex.MustCompile(`(^|/)node_modules/`),
regex.MustCompile(`(^|/)\.yarn/releases/`),
regex.MustCompile(`(^|/)\.yarn/plugins/`),
regex.MustCompile(`(^|/)\.yarn/sdks/`),
regex.MustCompile(`(^|/)\.yarn/versions/`),
regex.MustCompile(`(^|/)\.yarn/unplugged/`),
regex.MustCompile(`(^|/)_esy$`),
regex.MustCompile(`(^|/)bower_components/`),
regex.MustCompile(`^rebar$`),
regex.MustCompile(`(^|/)erlang\.mk`),
regex.MustCompile(`(^|/)Godeps/_workspace/`),
regex.MustCompile(`(^|/)testdata/`),
regex.MustCompile(`(^|/)\.indent\.pro`),
regex.MustCompile(`(\.|-)min\.(js|css)$`),
regex.MustCompile(`([^\s]*)import\.(css|less|scss|styl)$`),
regex.MustCompile(`(^|/)bootstrap([^/.]*)(\..*)?\.(js|css|less|scss|styl)$`),
regex.MustCompile(`(^|/)custom\.bootstrap([^\s]*)(js|css|less|scss|styl)$`),
regex.MustCompile(`(^|/)font-?awesome\.(css|less|scss|styl)$`),
regex.MustCompile(`(^|/)font-?awesome/.*\.(css|less|scss|styl)$`),
regex.MustCompile(`(^|/)foundation\.(css|less|scss|styl)$`),
regex.MustCompile(`(^|/)normalize\.(css|less|scss|styl)$`),
regex.MustCompile(`(^|/)skeleton\.(css|less|scss|styl)$`),
regex.MustCompile(`(^|/)[Bb]ourbon/.*\.(css|less|scss|styl)$`),
regex.MustCompile(`(^|/)animate\.(css|less|scss|styl)$`),
regex.MustCompile(`(^|/)materialize\.(css|less|scss|styl|js)$`),
regex.MustCompile(`(^|/)select2/.*\.(css|scss|js)$`),
regex.MustCompile(`(^|/)bulma\.(css|sass|scss)$`),
regex.MustCompile(`(3rd|[Tt]hird)[-_]?[Pp]arty/`),
regex.MustCompile(`(^|/)vendors?/`),
regex.MustCompile(`(^|/)[Ee]xtern(als?)?/`),
regex.MustCompile(`(^|/)[Vv]+endor/`),
regex.MustCompile(`^debian/`),
regex.MustCompile(`(^|/)run\.n$`),
regex.MustCompile(`(^|/)bootstrap-datepicker/`),
regex.MustCompile(`(^|/)jquery([^.]*)\.js$`),
regex.MustCompile(`(^|/)jquery\-\d\.\d+(\.\d+)?\.js$`),
regex.MustCompile(`(^|/)jquery\-ui(\-\d\.\d+(\.\d+)?)?(\.\w+)?\.(js|css)$`),
regex.MustCompile(`(^|/)jquery\.(ui|effects)\.([^.]*)\.(js|css)$`),
regex.MustCompile(`(^|/)jquery\.fn\.gantt\.js`),
regex.MustCompile(`(^|/)jquery\.fancybox\.(js|css)`),
regex.MustCompile(`(^|/)fuelux\.js`),
regex.MustCompile(`(^|/)jquery\.fileupload(-\w+)?\.js$`),
regex.MustCompile(`(^|/)jquery\.dataTables\.js`),
regex.MustCompile(`(^|/)bootbox\.js`),
regex.MustCompile(`(^|/)pdf\.worker\.js`),
regex.MustCompile(`(^|/)slick\.\w+.js$`),
regex.MustCompile(`(^|/)Leaflet\.Coordinates-\d+\.\d+\.\d+\.src\.js$`),
regex.MustCompile(`(^|/)leaflet\.draw-src\.js`),
regex.MustCompile(`(^|/)leaflet\.draw\.css`),
regex.MustCompile(`(^|/)Control\.FullScreen\.css`),
regex.MustCompile(`(^|/)Control\.FullScreen\.js`),
regex.MustCompile(`(^|/)leaflet\.spin\.js`),
regex.MustCompile(`(^|/)wicket-leaflet\.js`),
regex.MustCompile(`(^|/)\.sublime-project`),
regex.MustCompile(`(^|/)\.sublime-workspace`),
regex.MustCompile(`(^|/)\.vscode/`),
regex.MustCompile(`(^|/)prototype(.*)\.js$`),
regex.MustCompile(`(^|/)effects\.js$`),
regex.MustCompile(`(^|/)controls\.js$`),
regex.MustCompile(`(^|/)dragdrop\.js$`),
regex.MustCompile(`(.*?)\.d\.ts$`),
regex.MustCompile(`(^|/)mootools([^.]*)\d+\.\d+.\d+([^.]*)\.js$`),
regex.MustCompile(`(^|/)dojo\.js$`),
regex.MustCompile(`(^|/)MochiKit\.js$`),
regex.MustCompile(`(^|/)yahoo-([^.]*)\.js$`),
regex.MustCompile(`(^|/)yui([^.]*)\.js$`),
regex.MustCompile(`(^|/)ckeditor\.js$`),
regex.MustCompile(`(^|/)tiny_mce([^.]*)\.js$`),
regex.MustCompile(`(^|/)tiny_mce/(langs|plugins|themes|utils)`),
regex.MustCompile(`(^|/)ace-builds/`),
regex.MustCompile(`(^|/)fontello(.*?)\.css$`),
regex.MustCompile(`(^|/)MathJax/`),
regex.MustCompile(`(^|/)Chart\.js$`),
regex.MustCompile(`(^|/)[Cc]ode[Mm]irror/(\d+\.\d+/)?(lib|mode|theme|addon|keymap|demo)`),
regex.MustCompile(`(^|/)shBrush([^.]*)\.js$`),
regex.MustCompile(`(^|/)shCore\.js$`),
regex.MustCompile(`(^|/)shLegacy\.js$`),
regex.MustCompile(`(^|/)angular([^.]*)\.js$`),
regex.MustCompile(`(^|\/)d3(\.v\d+)?([^.]*)\.js$`),
regex.MustCompile(`(^|/)react(-[^.]*)?\.js$`),
regex.MustCompile(`(^|/)flow-typed/.*\.js$`),
regex.MustCompile(`(^|/)modernizr\-\d\.\d+(\.\d+)?\.js$`),
regex.MustCompile(`(^|/)modernizr\.custom\.\d+\.js$`),
regex.MustCompile(`(^|/)knockout-(\d+\.){3}(debug\.)?js$`),
regex.MustCompile(`(^|/)docs?/_?(build|themes?|templates?|static)/`),
regex.MustCompile(`(^|/)admin_media/`),
regex.MustCompile(`(^|/)env/`),
regex.MustCompile(`(^|/)fabfile\.py$`),
regex.MustCompile(`(^|/)waf$`),
regex.MustCompile(`(^|/)\.osx$`),
regex.MustCompile(`\.xctemplate/`),
regex.MustCompile(`\.imageset/`),
regex.MustCompile(`(^|/)Carthage/`),
regex.MustCompile(`(^|/)Sparkle/`),
regex.MustCompile(`(^|/)Crashlytics\.framework/`),
regex.MustCompile(`(^|/)Fabric\.framework/`),
regex.MustCompile(`(^|/)BuddyBuildSDK\.framework/`),
regex.MustCompile(`(^|/)Realm\.framework`),
regex.MustCompile(`(^|/)RealmSwift\.framework`),
regex.MustCompile(`(^|/)\.gitattributes$`),
regex.MustCompile(`(^|/)\.gitignore$`),
regex.MustCompile(`(^|/)\.gitmodules$`),
regex.MustCompile(`(^|/)gradlew$`),
regex.MustCompile(`(^|/)gradlew\.bat$`),
regex.MustCompile(`(^|/)gradle/wrapper/`),
regex.MustCompile(`(^|/)mvnw$`),
regex.MustCompile(`(^|/)mvnw\.cmd$`),
regex.MustCompile(`(^|/)\.mvn/wrapper/`),
regex.MustCompile(`-vsdoc\.js$`),
regex.MustCompile(`\.intellisense\.js$`),
regex.MustCompile(`(^|/)jquery([^.]*)\.validate(\.unobtrusive)?\.js$`),
regex.MustCompile(`(^|/)jquery([^.]*)\.unobtrusive\-ajax\.js$`),
regex.MustCompile(`(^|/)[Mm]icrosoft([Mm]vc)?([Aa]jax|[Vv]alidation)(\.debug)?\.js$`),
regex.MustCompile(`(^|/)[Pp]ackages\/.+\.\d+\/`),
regex.MustCompile(`(^|/)extjs/.*?\.js$`),
regex.MustCompile(`(^|/)extjs/.*?\.xml$`),
regex.MustCompile(`(^|/)extjs/.*?\.txt$`),
regex.MustCompile(`(^|/)extjs/.*?\.html$`),
regex.MustCompile(`(^|/)extjs/.*?\.properties$`),
regex.MustCompile(`(^|/)extjs/\.sencha/`),
regex.MustCompile(`(^|/)extjs/docs/`),
regex.MustCompile(`(^|/)extjs/builds/`),
regex.MustCompile(`(^|/)extjs/cmd/`),
regex.MustCompile(`(^|/)extjs/examples/`),
regex.MustCompile(`(^|/)extjs/locale/`),
regex.MustCompile(`(^|/)extjs/packages/`),
regex.MustCompile(`(^|/)extjs/plugins/`),
regex.MustCompile(`(^|/)extjs/resources/`),
regex.MustCompile(`(^|/)extjs/src/`),
regex.MustCompile(`(^|/)extjs/welcome/`),
regex.MustCompile(`(^|/)html5shiv\.js$`),
regex.MustCompile(`(^|/)[Tt]ests?/fixtures/`),
regex.MustCompile(`(^|/)[Ss]pecs?/fixtures/`),
regex.MustCompile(`(^|/)cordova([^.]*)\.js$`),
regex.MustCompile(`(^|/)cordova\-\d\.\d(\.\d)?\.js$`),
regex.MustCompile(`(^|/)foundation(\..*)?\.js$`),
regex.MustCompile(`(^|/)Vagrantfile$`),
regex.MustCompile(`(^|/)\.[Dd][Ss]_[Ss]tore$`),
regex.MustCompile(`(^|/)inst/extdata/`),
regex.MustCompile(`(^|/)octicons\.css`),
regex.MustCompile(`(^|/)sprockets-octicons\.scss`),
regex.MustCompile(`(^|/)activator$`),
regex.MustCompile(`(^|/)activator\.bat$`),
regex.MustCompile(`(^|/)proguard\.pro$`),
regex.MustCompile(`(^|/)proguard-rules\.pro$`),
regex.MustCompile(`(^|/)puphpet/`),
regex.MustCompile(`(^|/)\.google_apis/`),
regex.MustCompile(`(^|/)Jenkinsfile$`),
regex.MustCompile(`(^|/)\.gitpod\.Dockerfile$`),
regex.MustCompile(`(^|/)\.github/`),
regex.MustCompile(`(^|/)\.obsidian/`),
regex.MustCompile(`(^|/)\.teamcity/`),
}
// FastVendorMatcher is equivalent to matching any of the VendorMatchers.
var FastVendorMatcher = regex.MustCompile(`(?:^(?:(?:[Dd]ependencies/)|(?:debian/)|(?:deps/)|(?:rebar$)))|(?:(?:^|/)(?:(?:BuddyBuildSDK\.framework/)|(?:Carthage/)|(?:Chart\.js$)|(?:Control\.FullScreen\.css)|(?:Control\.FullScreen\.js)|(?:Crashlytics\.framework/)|(?:Fabric\.framework/)|(?:Godeps/_workspace/)|(?:Jenkinsfile$)|(?:Leaflet\.Coordinates-\d+\.\d+\.\d+\.src\.js$)|(?:MathJax/)|(?:MochiKit\.js$)|(?:RealmSwift\.framework)|(?:Realm\.framework)|(?:Sparkle/)|(?:Vagrantfile$)|(?:[Bb]ourbon/.*\.(css|less|scss|styl)$)|(?:[Cc]ode[Mm]irror/(\d+\.\d+/)?(lib|mode|theme|addon|keymap|demo))|(?:[Ee]xtern(als?)?/)|(?:[Mm]icrosoft([Mm]vc)?([Aa]jax|[Vv]alidation)(\.debug)?\.js$)|(?:[Pp]ackages\/.+\.\d+\/)|(?:[Ss]pecs?/fixtures/)|(?:[Tt]ests?/fixtures/)|(?:[Vv]+endor/)|(?:\.[Dd][Ss]_[Ss]tore$)|(?:\.gitattributes$)|(?:\.github/)|(?:\.gitignore$)|(?:\.gitmodules$)|(?:\.gitpod\.Dockerfile$)|(?:\.google_apis/)|(?:\.indent\.pro)|(?:\.mvn/wrapper/)|(?:\.obsidian/)|(?:\.osx$)|(?:\.sublime-project)|(?:\.sublime-workspace)|(?:\.teamcity/)|(?:\.vscode/)|(?:\.yarn/plugins/)|(?:\.yarn/releases/)|(?:\.yarn/sdks/)|(?:\.yarn/unplugged/)|(?:\.yarn/versions/)|(?:_esy$)|(?:ace-builds/)|(?:aclocal\.m4)|(?:activator$)|(?:activator\.bat$)|(?:admin_media/)|(?:angular([^.]*)\.js$)|(?:animate\.(css|less|scss|styl)$)|(?:bootbox\.js)|(?:bootstrap([^/.]*)(\..*)?\.(js|css|less|scss|styl)$)|(?:bootstrap-datepicker/)|(?:bower_components/)|(?:bulma\.(css|sass|scss)$)|(?:cache/)|(?:ckeditor\.js$)|(?:config\.guess$)|(?:config\.sub$)|(?:configure$)|(?:controls\.js$)|(?:cordova([^.]*)\.js$)|(?:cordova\-\d\.\d(\.\d)?\.js$)|(?:cpplint\.py)|(?:custom\.bootstrap([^\s]*)(js|css|less|scss|styl)$)|(?:dist/)|(?:docs?/_?(build|themes?|templates?|static)/)|(?:dojo\.js$)|(?:dotnet-install\.(ps1|sh)$)|(?:dragdrop\.js$)|(?:effects\.js$)|(?:env/)|(?:erlang\.mk)|(?:extjs/.*?\.html$)|(?:extjs/.*?\.js$)|(?:extjs/.*?\.properties$)|(?:extjs/.*?\.txt$)|(?:extjs/.*?\.xml$)|(?:extjs/\.sencha/)|(?:extjs/builds/)|(?:extjs/cmd/)|(?:extjs/docs/)|(?:extjs/examples/)|(?:extjs/locale/)|(?:extjs/packages/)|(?:extjs/plugins/)|(?:extjs/resources/)|(?:extjs/src/)|(?:extjs/welcome/)|(?:fabfile\.py$)|(?:flow-typed/.*\.js$)|(?:font-?awesome/.*\.(css|less|scss|styl)$)|(?:font-?awesome\.(css|less|scss|styl)$)|(?:fontello(.*?)\.css$)|(?:foundation(\..*)?\.js$)|(?:foundation\.(css|less|scss|styl)$)|(?:fuelux\.js)|(?:gradle/wrapper/)|(?:gradlew$)|(?:gradlew\.bat$)|(?:html5shiv\.js$)|(?:inst/extdata/)|(?:jquery([^.]*)\.js$)|(?:jquery([^.]*)\.unobtrusive\-ajax\.js$)|(?:jquery([^.]*)\.validate(\.unobtrusive)?\.js$)|(?:jquery\-\d\.\d+(\.\d+)?\.js$)|(?:jquery\-ui(\-\d\.\d+(\.\d+)?)?(\.\w+)?\.(js|css)$)|(?:jquery\.(ui|effects)\.([^.]*)\.(js|css)$)|(?:jquery\.dataTables\.js)|(?:jquery\.fancybox\.(js|css))|(?:jquery\.fileupload(-\w+)?\.js$)|(?:jquery\.fn\.gantt\.js)|(?:knockout-(\d+\.){3}(debug\.)?js$)|(?:leaflet\.draw-src\.js)|(?:leaflet\.draw\.css)|(?:leaflet\.spin\.js)|(?:libtool\.m4)|(?:ltoptions\.m4)|(?:ltsugar\.m4)|(?:ltversion\.m4)|(?:lt~obsolete\.m4)|(?:materialize\.(css|less|scss|styl|js)$)|(?:modernizr\-\d\.\d+(\.\d+)?\.js$)|(?:modernizr\.custom\.\d+\.js$)|(?:mootools([^.]*)\d+\.\d+.\d+([^.]*)\.js$)|(?:mvnw$)|(?:mvnw\.cmd$)|(?:node_modules/)|(?:normalize\.(css|less|scss|styl)$)|(?:octicons\.css)|(?:pdf\.worker\.js)|(?:proguard-rules\.pro$)|(?:proguard\.pro$)|(?:prototype(.*)\.js$)|(?:puphpet/)|(?:react(-[^.]*)?\.js$)|(?:run\.n$)|(?:select2/.*\.(css|scss|js)$)|(?:shBrush([^.]*)\.js$)|(?:shCore\.js$)|(?:shLegacy\.js$)|(?:skeleton\.(css|less|scss|styl)$)|(?:slick\.\w+.js$)|(?:sprockets-octicons\.scss)|(?:testdata/)|(?:tiny_mce([^.]*)\.js$)|(?:tiny_mce/(langs|plugins|themes|utils))|(?:vendors?/)|(?:waf$)|(?:wicket-leaflet\.js)|(?:yahoo-([^.]*)\.js$)|(?:yui([^.]*)\.js$)))|(?:(.*?)\.d\.ts$)|(?:(3rd|[Tt]hird)[-_]?[Pp]arty/)|(?:([^\s]*)import\.(css|less|scss|styl)$)|(?:(\.|-)min\.(js|css)$)|(?:(^|\/)d3(\.v\d+)?([^.]*)\.js$)|(?:-vsdoc\.js$)|(?:\.imageset/)|(?:\.intellisense\.js$)|(?:\.xctemplate/)`)

30
enry.go Normal file
View File

@ -0,0 +1,30 @@
/*
Package enry identifies programming languages.
Identification is based on file name and content using a series
of strategies to narrow down possible options.
Each strategy is available as a separate API call, as well as though the main enty point:
GetLanguage(filename string, content []byte) (language string)
It is a port of the https://github.com/github/linguist from Ruby.
Upstream Linguist YAML files are used to generate datastructures for data
package.
*/
package enry // import "github.com/go-enry/go-enry/v2"
//go:generate make code-generate
import "github.com/go-enry/go-enry/v2/data"
// Type represent language's type. Either data, programming, markup, prose, or unknown.
type Type int
// Type's values.
const (
Unknown Type = Type(data.TypeUnknown)
Data = Type(data.TypeData)
Programming = Type(data.TypeProgramming)
Markup = Type(data.TypeMarkup)
Prose = Type(data.TypeProse)
)

9
go.mod Normal file
View File

@ -0,0 +1,9 @@
module github.com/go-enry/go-enry/v2
go 1.14
require (
github.com/go-enry/go-oniguruma v1.2.1
github.com/stretchr/testify v1.8.1
gopkg.in/yaml.v2 v2.2.8
)

21
go.sum Normal file
View File

@ -0,0 +1,21 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo=
github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

Some files were not shown because too many files have changed in this diff Show More