From fd84042220439d62a41a0edec8057084d5941459 Mon Sep 17 00:00:00 2001 From: Sven Rebhan <36194019+srebhan@users.noreply.github.com> Date: Thu, 28 Jul 2022 16:21:07 +0200 Subject: [PATCH] feat: Add license checking tool (#11398) --- .circleci/config.yml | 16 ++ .gitignore | 1 + Makefile | 3 + docs/LICENSE_OF_DEPENDENCIES.md | 16 +- go.mod | 3 +- go.sum | 2 + tools/license_checker/README.md | 93 +++++++ tools/license_checker/data/spdx_mapping.json | 15 ++ tools/license_checker/data/whitelist | 3 + tools/license_checker/main.go | 246 +++++++++++++++++++ tools/license_checker/package.go | 97 ++++++++ tools/license_checker/whitelist.go | 119 +++++++++ 12 files changed, 605 insertions(+), 9 deletions(-) create mode 100644 tools/license_checker/README.md create mode 100644 tools/license_checker/data/spdx_mapping.json create mode 100644 tools/license_checker/data/whitelist create mode 100644 tools/license_checker/main.go create mode 100644 tools/license_checker/package.go create mode 100644 tools/license_checker/whitelist.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 5275b0397..bebdc4e91 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -232,6 +232,21 @@ jobs: os: windows gotestsum: "gotestsum.exe" + test-licenses: + executor: telegraf-ci + steps: + - checkout + - restore_cache: + key: go-mod-v1-{{ checksum "go.sum" }} + - check-changed-files-or-halt + - run: 'make build_tools' + - run: 'sh ./tools/license_checker/license_checker -whitelist ./tools/license_checker/data/whitelist' + - save_cache: + name: 'go module cache' + key: go-mod-v1-{{ checksum "go.sum" }} + paths: + - '/go/pkg/mod' + windows-package: parameters: nightly: @@ -725,6 +740,7 @@ workflows: - 'test-go-linux-386' - 'test-go-mac' - 'test-go-windows' + - 'test-licenses' - 'windows-package': name: 'windows-package-nightly' nightly: true diff --git a/.gitignore b/.gitignore index 293407ff4..e73cac009 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ /telegraf.exe /telegraf.gz /tools/package_lxd_test/package_lxd_test +/tools/license_checker/license_checker /tools/readme_config_includer/generator /vendor .DS_Store diff --git a/Makefile b/Makefile index 9748a1d6d..59c203f2e 100644 --- a/Makefile +++ b/Makefile @@ -115,6 +115,7 @@ versioninfo: go generate cmd/telegraf/telegraf_windows.go; \ build_tools: + $(HOSTGO) build -o ./tools/license_checker/license_checker ./tools/license_checker $(HOSTGO) build -o ./tools/readme_config_includer/generator ./tools/readme_config_includer/generator.go embed_readme_%: @@ -227,6 +228,8 @@ clean: rm -rf tools/readme_config_includer/generator.exe rm -rf tools/package_lxd_test/package_lxd_test rm -rf tools/package_lxd_test/package_lxd_test.exe + rm -rf tools/license_checker/license_checker + rm -rf tools/license_checker/license_checker.exe .PHONY: docker-image docker-image: diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index 934abc11d..f8af68c85 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -10,7 +10,7 @@ following works: - github.com/Azure/azure-event-hubs-go [MIT License](https://github.com/Azure/azure-event-hubs-go/blob/master/LICENSE) - github.com/Azure/azure-kusto-go [MIT License](https://github.com/Azure/azure-kusto-go/blob/master/LICENSE) - github.com/Azure/azure-pipeline-go [MIT License](https://github.com/Azure/azure-pipeline-go/blob/master/LICENSE) -- github.com/Azure/azure-sdk-for-go [Apache License 2.0](https://github.com/Azure/azure-sdk-for-go/blob/master/LICENSE) +- github.com/Azure/azure-sdk-for-go [MIT License](https://github.com/Azure/azure-sdk-for-go/blob/main/LICENSE.txt) - github.com/Azure/azure-sdk-for-go/sdk/azcore [MIT License](https://github.com/Azure/azure-sdk-for-go/blob/main/sdk/azcore/LICENSE.txt) - github.com/Azure/azure-sdk-for-go/sdk/internal [MIT License](https://github.com/Azure/azure-sdk-for-go/blob/main/sdk/internal/LICENSE.txt) - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob [MIT License](https://github.com/Azure/azure-sdk-for-go/blob/main/sdk/storage/azblob/LICENSE.txt) @@ -143,12 +143,12 @@ following works: - github.com/grid-x/serial [MIT License](https://github.com/grid-x/serial/blob/master/LICENSE) - github.com/gwos/tcg/sdk [MIT License](https://github.com/gwos/tcg/blob/master/LICENSE) - github.com/hailocab/go-hostpool [MIT License](https://github.com/hailocab/go-hostpool/blob/master/LICENSE) -- github.com/harlow/kinesis-consumer [MIT License](https://github.com/harlow/kinesis-consumer/blob/master/MIT-LICENSE) +- github.com/harlow/kinesis-consumer [MIT License](https://github.com/harlow/kinesis-consumer/blob/master/LICENSE) - github.com/hashicorp/consul/api [Mozilla Public License 2.0](https://github.com/hashicorp/consul/blob/master/LICENSE) - github.com/hashicorp/errwrap [Mozilla Public License 2.0](https://github.com/hashicorp/errwrap/blob/master/LICENSE) - github.com/hashicorp/go-cleanhttp [Mozilla Public License 2.0](https://github.com/hashicorp/go-cleanhttp/blob/master/LICENSE) -- github.com/hashicorp/go-hclog [Mozilla Public License 2.0](https://github.com/hashicorp/go-hclog/LICENSE) -- github.com/hashicorp/go-immutable-radix [Mozilla Public License 2.0](https://github.com/hashicorp/go-immutable-radix/LICENSE) +- github.com/hashicorp/go-hclog [MIT License](https://github.com/hashicorp/go-hclog/blob/main/LICENSE) +- github.com/hashicorp/go-immutable-radix [Mozilla Public License 2.0](https://github.com/hashicorp/go-immutable-radix/blob/master/LICENSE) - github.com/hashicorp/go-multierror [Mozilla Public License 2.0](https://github.com/hashicorp/go-multierror/blob/master/LICENSE) - github.com/hashicorp/go-rootcerts [Mozilla Public License 2.0](https://github.com/hashicorp/go-rootcerts/blob/master/LICENSE) - github.com/hashicorp/go-uuid [Mozilla Public License 2.0](https://github.com/hashicorp/go-uuid/blob/master/LICENSE) @@ -180,7 +180,7 @@ following works: - github.com/jcmturner/rpc [Apache License 2.0](https://github.com/jcmturner/rpc/blob/master/LICENSE) - github.com/jhump/protoreflect [Apache License 2.0](https://github.com/jhump/protoreflect/blob/master/LICENSE) - github.com/jmespath/go-jmespath [Apache License 2.0](https://github.com/jmespath/go-jmespath/blob/master/LICENSE) -- github.com/josharian/intern [MIT License](https://github.com/josharian/intern/blob/master/license.md) +- github.com/josharian/intern [MIT License](https://github.com/josharian/intern/blob/master/LICENSE.md) - github.com/josharian/native [MIT License](https://github.com/josharian/native/blob/main/license) - github.com/jpillora/backoff [MIT License](https://github.com/jpillora/backoff/blob/master/LICENSE) - github.com/json-iterator/go [MIT License](https://github.com/json-iterator/go/blob/master/LICENSE) @@ -244,7 +244,7 @@ following works: - github.com/prometheus/procfs [Apache License 2.0](https://github.com/prometheus/procfs/blob/master/LICENSE) - github.com/prometheus/prometheus [Apache License 2.0](https://github.com/prometheus/prometheus/blob/master/LICENSE) - github.com/rabbitmq/amqp091-go [BSD 2-Clause "Simplified" License](https://github.com/rabbitmq/amqp091-go/blob/main/LICENSE) -- github.com/rcrowley/go-metrics [MIT License](https://github.com/rcrowley/go-metrics/blob/master/LICENSE) +- github.com/rcrowley/go-metrics [BSD 2-Clause with views sentence](https://github.com/rcrowley/go-metrics/blob/master/LICENSE) - github.com/remyoudompheng/bigfft [BSD 3-Clause "New" or "Revised" License](https://github.com/remyoudompheng/bigfft/blob/master/LICENSE) - github.com/riemann/riemann-go-client [MIT License](https://github.com/riemann/riemann-go-client/blob/master/LICENSE) - github.com/robbiet480/go.nut [MIT License](https://github.com/robbiet480/go.nut/blob/master/LICENSE) @@ -316,7 +316,7 @@ following works: - gopkg.in/tomb.v1 [BSD 3-Clause Clear License](https://github.com/go-tomb/tomb/blob/v1/LICENSE) - gopkg.in/tomb.v2 [BSD 3-Clause Clear License](https://github.com/go-tomb/tomb/blob/v2/LICENSE) - gopkg.in/yaml.v2 [Apache License 2.0](https://github.com/go-yaml/yaml/blob/v2.2.2/LICENSE) -- gopkg.in/yaml.v3 [Apache License 2.0](https://github.com/go-yaml/yaml/blob/v3/LICENSE) +- gopkg.in/yaml.v3 [MIT License](https://github.com/go-yaml/yaml/blob/v3/LICENSE) - k8s.io/api [Apache License 2.0](https://github.com/kubernetes/client-go/blob/master/LICENSE) - k8s.io/apimachinery [Apache License 2.0](https://github.com/kubernetes/client-go/blob/master/LICENSE) - k8s.io/client-go [Apache License 2.0](https://github.com/kubernetes/client-go/blob/master/LICENSE) @@ -333,4 +333,4 @@ following works: ## Telegraf used and modified code from these projects -- github.com/DataDog/datadog-agent [Apache License 2.0](https://github.com/DataDog/datadog-agent/LICENSE) +- github.com/DataDog/datadog-agent [Apache License 2.0](https://github.com/DataDog/datadog-agent/blob/main/LICENSE) diff --git a/go.mod b/go.mod index 941679691..b0bbfdff6 100644 --- a/go.mod +++ b/go.mod @@ -74,6 +74,7 @@ require ( github.com/google/gnxi v0.0.0-20220411075422-cd6b043b7fd0 github.com/google/go-cmp v0.5.8 github.com/google/go-github/v32 v32.1.0 + github.com/google/licensecheck v0.3.1 github.com/google/uuid v1.3.0 github.com/gopcua/opcua v0.3.3 github.com/gophercloud/gophercloud v0.25.0 @@ -155,6 +156,7 @@ require ( go.opentelemetry.io/otel/metric v0.30.0 go.opentelemetry.io/otel/sdk/metric v0.28.0 go.starlark.net v0.0.0-20220328144851-d1966c6b9fcd + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 golang.org/x/net v0.0.0-20220622184535-263ec571b305 golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f @@ -375,7 +377,6 @@ require ( go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect golang.org/x/tools v0.1.11 // indirect diff --git a/go.sum b/go.sum index 850688f60..62bbff49e 100644 --- a/go.sum +++ b/go.sum @@ -1105,6 +1105,8 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/licensecheck v0.3.1 h1:QoxgoDkaeC4nFrtGN1jV7IPmDCHFNIVh54e5hSt6sPs= +github.com/google/licensecheck v0.3.1/go.mod h1:ORkR35t/JjW+emNKtfJDII0zlciG9JgbT7SmsohlHmY= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= diff --git a/tools/license_checker/README.md b/tools/license_checker/README.md new file mode 100644 index 000000000..c06b127b6 --- /dev/null +++ b/tools/license_checker/README.md @@ -0,0 +1,93 @@ +# Dependency license verification tool + +This tool allows the verification of information in +`docs/LICENSE_OF_DEPENDENCIES.md` against the linked license +information. To do so, the license reported by the user is +checked against the license classification of the downloaded +license file for each dependency. + +## Building + +```shell +make build_tools +``` + +## Running + +The simplest way to run the verification tool is to execute + +```shell +telegraf$ ./tools/license_checker/license_checker +``` + +using the current directory as telegraf's root directory and verifies +all licenses. Only errors will be reported by default. + +There are multiple options you can use to customize the verification. +Take a look at + +```shell +telegraf$ ./tools/license_checker/license_checker --help +``` + +to get an overview. + +As the verification tool downloads each license file linked in the +dependency license document, you should be careful on not exceeding +the access limits of e.g. GitHub by running the tool too frequent. + +Some packages change the license for newer versions. As we always +link to the latest license text the classification might not match +the actual license of our used dependency. Furthermore, some license +text might be wrongly classified, or not classified at all. In these +cases, you can use a _whitelist_ to explicitly state the license +SPDX classifier for those packages. +See the [whitelist section](#whitelist) for more details. + +The recommended use in telegraf is to run + +```shell +telegraf$ ./tools/license_checker/license_checker \ + -whitelist ./tools/license_checker/data/whitelist +``` + +using the code-versioned whitelist. This command will report all +non-matching entries with an `ERR:` prefix. + +## Whitelist + +Whitelist entries contain explicit license information for +a set of packages to use instead of classification. Each entry +in the whitelist is a line of the form + +```text +[comparison operator][@vX.Y.Z] +``` + +where the _comparison operator_ is one of `>`, `>=`, `=`, `<=` or `<` +and the _license SPDX_ is a [SPDX license identifier][spdx]. +In case no package version is specified, the entry matches all versions +of the library. Furthermore, the comparison operator can be omitted +which is equivalent to an exact match (`=`). + +The entries are processed in order until the first match is found. + +Here is an example of a whitelist. Assume that you have library +`github.com/foo/bar` which started out with the `MIT` license +until version 1.0.0 where it changed to `EFL-1.0` until it again +changed to `EFL-2.0` starting __after__ version 2.3.0. In this case +the whitelist should look like this + +```text + 1 { + //nolint:revive // We cannot do anything about possible failures here + _, _ = fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s [options] [telegraf root dir]\n", os.Args[0]) + _, _ = fmt.Fprintf(flag.CommandLine.Output(), "Options:\n") + flag.PrintDefaults() + _, _ = fmt.Fprintf(flag.CommandLine.Output(), "\n") + _, _ = fmt.Fprintf(flag.CommandLine.Output(), "Arguments:\n") + _, _ = fmt.Fprintf(flag.CommandLine.Output(), " telegraf root dir (optional)\n") + _, _ = fmt.Fprintf(flag.CommandLine.Output(), " path to the root directory of telegraf (default: .)\n") + os.Exit(1) + } + + // Setup full-name to license SPDX identifier mapping + if err := json.Unmarshal(spdxMappingFile, &nameToSPDX); err != nil { + log.Fatalf("Unmarshalling license name to SPDX mapping failed: %v", err) + } + + // Get required files + path := "." + if flag.NArg() == 1 { + path = flag.Arg(0) + } + + moduleFilename := filepath.Join(path, "go.mod") + licenseFilename := filepath.Join(path, "docs", "LICENSE_OF_DEPENDENCIES.md") + + var override whitelist + if whitelistFn != "" { + log.Printf("Reading whitelist file %q...", whitelistFn) + if err := override.Parse(whitelistFn); err != nil { + log.Fatalf("Reading whitelist failed: %v", err) + } + } + + log.Printf("Reading module file %q...", moduleFilename) + modbuf, err := os.ReadFile(moduleFilename) + if err != nil { + log.Fatal(err) + } + depModules, err := modfile.Parse(moduleFilename, modbuf, nil) + if err != nil { + log.Fatalf("Parsing modules failed: %f", err) + } + debugf("found %d required packages", len(depModules.Require)) + + dependencies := make(map[string]string) + for _, d := range depModules.Require { + dependencies[d.Mod.Path] = d.Mod.Version + } + + log.Printf("Reading license file %q...", licenseFilename) + licensesMarkdown, err := os.ReadFile(licenseFilename) + if err != nil { + log.Fatal(err) + } + + // Parse the markdown document + parser := goldmark.DefaultParser() + root := parser.Parse(text.NewReader(licensesMarkdown)) + + // Prepare a line parser + lineParser := goldmark.DefaultParser() + + // Collect the licenses + // For each list we search for the items and parse them. + // Expect a pattern of . + ignored := 0 + var packageInfos []packageInfo + for node := root.FirstChild(); node != nil; node = node.NextSibling() { + listNode, ok := node.(*ast.List) + if !ok { + continue + } + + for inode := listNode.FirstChild(); inode != nil; inode = inode.NextSibling() { + itemNode, ok := inode.(*ast.ListItem) + if !ok || itemNode.ChildCount() != 1 { + continue + } + textNode, ok := itemNode.FirstChild().(*ast.TextBlock) + if !ok || textNode.Lines().Len() != 1 { + continue + } + + lineSegment := textNode.Lines().At(0) + line := lineSegment.Value(licensesMarkdown) + lineRoot := lineParser.Parse(text.NewReader(line)) + if lineRoot.ChildCount() != 1 || lineRoot.FirstChild().ChildCount() < 2 { + log.Printf("WARN: Ignoring item %q due to wrong count (%d/%d)", string(line), lineRoot.ChildCount(), lineRoot.FirstChild().ChildCount()) + ignored++ + continue + } + + var name, license, link string + for lineElementNode := lineRoot.FirstChild().FirstChild(); lineElementNode != nil; lineElementNode = lineElementNode.NextSibling() { + switch v := lineElementNode.(type) { + case *ast.Text: + name += string(v.Text(line)) + case *ast.Link: + license = string(v.Text(line)) + link = string(v.Destination) + default: + debugf("ignoring unknown element %T (%v)", v, v) + } + } + + info := packageInfo{ + name: strings.TrimSpace(name), + version: dependencies[name], + url: strings.TrimSpace(link), + license: strings.TrimSpace(license), + } + info.ToSPDX() + if info.name == "" { + log.Printf("WARN: Ignoring item %q due to empty package name", string(line)) + ignored++ + continue + } + if info.url == "" { + log.Printf("WARN: Ignoring item %q due to empty url name", string(line)) + ignored++ + continue + } + if info.license == "" { + log.Printf("WARN: Ignoring item %q due to empty license name", string(line)) + ignored++ + continue + } + debugf("adding %q with license %q (%s) and version %q at %q...", info.name, info.license, info.spdx, info.version, info.url) + packageInfos = append(packageInfos, info) + } + } + + // Get the superset of licenses + if debug { + licenseSet := map[string]bool{} + licenseNames := []string{} + for _, info := range packageInfos { + if found := licenseSet[info.license]; !found { + licenseNames = append(licenseNames, info.license) + } + licenseSet[info.license] = true + } + sort.Strings(licenseNames) + log.Println("Using licenses:") + for _, license := range licenseNames { + log.Println(" " + license) + } + } + + // Check the licenses by matching their text and compare the classification result + // with the information provided by the user + var succeeded, warn, failed int + for _, info := range packageInfos { + // Ignore all packages except the ones given by the user (if any) + if userpkg != "" && userpkg != info.name { + continue + } + + // Check if we got a whitelist entry for the package + if ok, found := override.Check(info.name, info.version, info.spdx); found { + if ok { + log.Printf("OK: %q (%s) (whitelist)", info.name, info.license) + succeeded++ + } else { + log.Printf("ERR: %q (%s) %s does not match whitelist", info.name, info.license, info.spdx) + failed++ + } + continue + } + + // Perform a text classification + confidence, err := info.Classify() + if err != nil { + log.Printf("ERR: %q (%s) %v", info.name, info.license, err) + failed++ + continue + } + if confidence < threshold { + log.Printf("WARN: %q (%s) has low matching confidence (%.2f%%)", info.name, info.license, confidence) + warn++ + continue + } + if verbose { + log.Printf("OK: %q (%s) (%.2f%%)", info.name, info.license, confidence) + } + succeeded++ + } + if verbose { + log.Printf("Checked %d licenses (%d ignored lines):", len(packageInfos), ignored) + log.Printf(" %d successful", succeeded) + log.Printf(" %d low confidence", warn) + log.Printf(" %d errors", failed) + } + + if failed > 0 { + os.Exit(1) + } + os.Exit(0) +} diff --git a/tools/license_checker/package.go b/tools/license_checker/package.go new file mode 100644 index 000000000..7e41d0dfb --- /dev/null +++ b/tools/license_checker/package.go @@ -0,0 +1,97 @@ +package main + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/google/licensecheck" +) + +type packageInfo struct { + name string + version string + license string + url string + spdx string +} + +func (pkg *packageInfo) ToSPDX() { + pkg.spdx = nameToSPDX[pkg.license] +} + +func (pkg *packageInfo) Classify() (float64, error) { + // Download the license text + source, err := normalizeURL(pkg.url) + if err != nil { + return 0.0, fmt.Errorf("%q is not a valid URL: %w", pkg.url, err) + } + debugf("%q downloading from %q", pkg.name, source) + + response, err := http.Get(source) + if err != nil { + return 0.0, fmt.Errorf("download from %q failed: %w", source, err) + } + if response.StatusCode < 200 || response.StatusCode > 299 { + status := response.StatusCode + return 0.0, fmt.Errorf("download from %q failed %d: %s", source, status, http.StatusText(status)) + } + defer response.Body.Close() + text, err := io.ReadAll(response.Body) + if err != nil { + return 0.0, fmt.Errorf("reading body failed: %w", err) + } + if len(text) < 1 { + return 0.0, errors.New("empty body") + } + + // Classify the license text + coverage := licensecheck.Scan(text) + if len(coverage.Match) == 0 { + return coverage.Percent, errors.New("no match found") + } + match := coverage.Match[0] + debugf("%q found match: %q with confidence %f%%", pkg.name, match.ID, coverage.Percent) + + if match.ID != pkg.spdx { + return coverage.Percent, fmt.Errorf("classification %q does not match", match.ID) + } + return coverage.Percent, nil +} + +func normalizeURL(raw string) (string, error) { + u, err := url.Parse(raw) + if err != nil { + return "", err + } + + switch u.Hostname() { + case "github.com": + u.Host = "raw.githubusercontent.com" + var cleaned []string + for _, p := range strings.Split(u.Path, "/") { + // Filter out elements + if p == "blob" { + continue + } + cleaned = append(cleaned, p) + } + u.Path = strings.Join(cleaned, "/") + case "gitlab.com": + u.Path = strings.Replace(u.Path, "/-/blob/", "/-/raw/", 1) + case "git.octo.it": + parts := strings.Split(u.RawQuery, ";") + for i, p := range parts { + if p == "a=blob" { + parts[i] = "a=blob_plain" + break + } + } + u.RawQuery = strings.Join(parts, ";") + } + + return u.String(), nil +} diff --git a/tools/license_checker/whitelist.go b/tools/license_checker/whitelist.go new file mode 100644 index 000000000..6eae522ff --- /dev/null +++ b/tools/license_checker/whitelist.go @@ -0,0 +1,119 @@ +package main + +import ( + "bufio" + "log" //nolint:revive // We cannot use the Telegraf's logging here + "os" + "regexp" + "strings" + + "github.com/coreos/go-semver/semver" +) + +type whitelist []whitelistEntry + +type whitelistEntry struct { + Name string + Version *semver.Version + Operator string + License string +} + +var re = regexp.MustCompile(`^([<=>]+\s*)?([-\.\/\w]+)(@v[\d\.]+)?\s+([-\.\w]+)$`) + +func (w *whitelist) Parse(filename string) error { + file, err := os.Open(filename) + if err != nil { + return err + } + defer file.Close() + + // Read file line-by-line and split by semicolon + lineno := 0 + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + lineno++ + if strings.HasPrefix(line, "#") { + // Comment + continue + } + + groups := re.FindAllStringSubmatch(line, -1) + if len(groups) != 1 { + log.Printf("WARN: Ignoring not matching entry in line %d", lineno) + continue + } + group := groups[0] + if len(group) != 5 { + // Malformed + log.Printf("WARN: Ignoring malformed entry in line %d", lineno) + continue + } + + // An entry has the form: + // [operator][@version] [license SPDX] + var operator, version string + if group[1] != "" { + operator = strings.TrimSpace(group[1]) + } + name := group[2] + if group[3] != "" { + version = strings.TrimSpace(group[3]) + version = strings.TrimLeft(version, "@v") + } + license := strings.TrimSpace(group[4]) + + entry := whitelistEntry{Name: name, License: license, Operator: operator} + if version != "" { + entry.Version, err = semver.NewVersion(version) + if err != nil { + // Malformed + log.Printf("Ignoring malformed version in line %d: %v", lineno, err) + continue + } + if entry.Operator == "" { + entry.Operator = "=" + } + } + *w = append(*w, entry) + } + + return scanner.Err() +} + +func (w *whitelist) Check(pkg, version, spdx string) (ok, found bool) { + var pkgver semver.Version + v := strings.TrimSpace(version) + v = strings.TrimPrefix(v, "v") + if v != "" { + pkgver = *semver.New(v) + } + + for _, entry := range *w { + if entry.Name != pkg { + continue + } + + var match bool + switch entry.Operator { + case "": + match = true + case "=": + match = pkgver.Equal(*entry.Version) + case "<": + match = pkgver.LessThan(*entry.Version) + case "<=": + match = pkgver.LessThan(*entry.Version) || pkgver.Equal(*entry.Version) + case ">": + match = !(pkgver.LessThan(*entry.Version) || pkgver.Equal(*entry.Version)) + case ">=": + match = !pkgver.LessThan(*entry.Version) + } + if match { + return entry.License == spdx, true + } + } + + return false, false +}