You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@skywalking.apache.org by ke...@apache.org on 2022/06/14 07:59:05 UTC
[skywalking-eyes] branch config/excludes created (now 141b781)
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a change to branch config/excludes
in repository https://gitbox.apache.org/repos/asf/skywalking-eyes.git
at 141b781 Add `excludes` to `license resolve` config
This branch includes the following new commits:
new 141b781 Add `excludes` to `license resolve` config
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[skywalking-eyes] 01/01: Add `excludes` to `license resolve` config
Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch config/excludes
in repository https://gitbox.apache.org/repos/asf/skywalking-eyes.git
commit 141b781379a2eefb57e76218a08d609727ab42de
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Tue Jun 14 15:58:55 2022 +0800
Add `excludes` to `license resolve` config
---
README.md | 8 ++++--
pkg/deps/config.go | 41 ++++++++++++++++++++++++++++++
pkg/deps/golang.go | 22 ++++++++--------
pkg/deps/maven.go | 73 ++++++++++++++++++++----------------------------------
pkg/deps/npm.go | 24 ++++++------------
pkg/deps/result.go | 8 ++++++
6 files changed, 100 insertions(+), 76 deletions(-)
diff --git a/README.md b/README.md
index 55d81d3..405d1d0 100644
--- a/README.md
+++ b/README.md
@@ -769,6 +769,9 @@ dependency: # <15>
version: dependency-version # <19>
license: Apache-2.0 # <20>
threshold: 75 # <21>
+ excludes: # <22>
+ - name: dependency-name # the same format as <18>
+ version: dependency-version # the same format as <19>
```
1. The `header` section is configurations for source codes license header.
@@ -788,10 +791,11 @@ dependency: # <15>
15. The `dependency` section is configurations for resolving dependencies' licenses.
16. The `files` are the files that declare the dependencies of a project, typically, `go.mod` in Go project, `pom.xml` in maven project, and `package.json` in NodeJS project. If it's a relative path, it's relative to the `.licenserc.yaml`.
17. Declare the licenses which cannot be identified by this tool.
-18. The `name` of the dependency, The name is different for different projects, `PackagePath` in Go project, `GroupID:ArtifactID` in maven project, `PackageName` in NodeJS project.
-19. The `version` of the dependency, it's locked, preventing license changed between different versions.
+18. The `name` of the dependency, The name is different for different projects, `PackagePath` in Go project, `GroupID:ArtifactID` in maven project, `PackageName` in NodeJS project. You can use file pattern as described in [the doc](https://pkg.go.dev/path/filepath#Match).
+19. The `version` of the dependency, comma seperated string (such as `1.0,2.0,3.0`), if this is empty, it means all versions of the dependency.
20. The [SPDX ID](https://spdx.org/licenses/) of the dependency license.
21. The minimum percentage of the file that must contain license text for identifying a license, default is `75`.
+22. The dependencies that should be excluded when analyzing the licenses, this is useful when you declare the dependencies in `pom.xml` with `compile` scope but don't distribute them in package. (Note that non-`compile` scope dependencies are automatically excluded so you don't need to put them here).
**NOTE**: When the `SPDX-ID` is Apache-2.0 and the owner is Apache Software foundation, the content would be [a dedicated license](https://www.apache.org/legal/src-headers.html#headers) specified by the ASF, otherwise, the license would be [the standard one](https://www.apache.org/foundation/license-faq.html#Apply-My-Software).
diff --git a/pkg/deps/config.go b/pkg/deps/config.go
index d3a77ed..c29d7de 100644
--- a/pkg/deps/config.go
+++ b/pkg/deps/config.go
@@ -20,6 +20,7 @@ package deps
import (
"os"
"path/filepath"
+ "strings"
)
// DefaultCoverageThreshold is the minimum percentage of the file
@@ -31,6 +32,7 @@ type ConfigDeps struct {
Threshold int `yaml:"threshold"`
Files []string `yaml:"files"`
Licenses []*ConfigDepLicense `yaml:"licenses"`
+ Excludes []Exclude `yaml:"excludes"`
}
type ConfigDepLicense struct {
@@ -39,6 +41,11 @@ type ConfigDepLicense struct {
License string `yaml:"license"`
}
+type Exclude struct {
+ Name string `yaml:"name"`
+ Version string `yaml:"version"`
+}
+
func (config *ConfigDeps) Finalize(configFile string) error {
configFileAbsPath, err := filepath.Abs(configFile)
if err != nil {
@@ -58,3 +65,37 @@ func (config *ConfigDeps) Finalize(configFile string) error {
return nil
}
+
+func (conf *ConfigDeps) GetUserConfiguredLicense(name, version string) (string, bool) {
+ for _, license := range conf.Licenses {
+ if matched, _ := filepath.Match(license.Name, name); !matched && license.Name != name {
+ continue
+ }
+ if license.Version == "" {
+ return license.License, true
+ }
+ for _, v := range strings.Split(license.Version, ",") {
+ if v == version {
+ return license.License, true
+ }
+ }
+ }
+ return "", false
+}
+
+func (conf *ConfigDeps) IsExcluded(name, version string) bool {
+ for _, license := range conf.Excludes {
+ if matched, _ := filepath.Match(license.Name, name); !matched && license.Name != name {
+ continue
+ }
+ if license.Version == "" {
+ return true
+ }
+ for _, v := range strings.Split(license.Version, ",") {
+ if v == version {
+ return true
+ }
+ }
+ }
+ return false
+}
diff --git a/pkg/deps/golang.go b/pkg/deps/golang.go
index 842be06..9b97f3f 100644
--- a/pkg/deps/golang.go
+++ b/pkg/deps/golang.go
@@ -27,7 +27,6 @@ import (
"os/exec"
"path/filepath"
"regexp"
- "strings"
"github.com/apache/skywalking-eyes/internal/logger"
"github.com/apache/skywalking-eyes/pkg/license"
@@ -86,17 +85,16 @@ func (resolver *GoModResolver) Resolve(goModFile string, config *ConfigDeps, rep
func (resolver *GoModResolver) ResolvePackages(modules []*packages.Module, config *ConfigDeps, report *Report) error {
for _, module := range modules {
func() {
- for _, l := range config.Licenses {
- for _, version := range strings.Split(l.Version, ",") {
- if l.Name == module.Path && version == module.Version {
- report.Resolve(&Result{
- Dependency: module.Path,
- LicenseSpdxID: l.License,
- Version: module.Version,
- })
- return
- }
- }
+ if config.IsExcluded(module.Path, module.Version) {
+ return
+ }
+ if license, ok := config.GetUserConfiguredLicense(module.Path, module.Version); ok {
+ report.Resolve(&Result{
+ Dependency: module.Path,
+ LicenseSpdxID: license,
+ Version: module.Version,
+ })
+ return
}
err := resolver.ResolvePackageLicense(config, module, report)
if err != nil {
diff --git a/pkg/deps/maven.go b/pkg/deps/maven.go
index 28c5c08..53b353d 100644
--- a/pkg/deps/maven.go
+++ b/pkg/deps/maven.go
@@ -58,14 +58,14 @@ func (resolver *MavenPomResolver) Resolve(mavenPomFile string, config *ConfigDep
return err
}
- deps, err := resolver.LoadDependencies()
+ deps, err := resolver.LoadDependencies(config)
if err != nil {
// attempt to download dependencies
if err = resolver.DownloadDeps(); err != nil {
return fmt.Errorf("dependencies download error")
}
// load again
- deps, err = resolver.LoadDependencies()
+ deps, err = resolver.LoadDependencies(config)
if err != nil {
return err
}
@@ -125,7 +125,7 @@ func (resolver *MavenPomResolver) DownloadDeps() error {
return install.Run()
}
-func (resolver *MavenPomResolver) LoadDependencies() ([]*Dependency, error) {
+func (resolver *MavenPomResolver) LoadDependencies(config *ConfigDeps) ([]*Dependency, error) {
buf := bytes.NewBuffer(nil)
cmd := exec.Command(resolver.maven, "dependency:tree") // #nosec G204
@@ -138,7 +138,7 @@ func (resolver *MavenPomResolver) LoadDependencies() ([]*Dependency, error) {
return nil, err
}
- deps := LoadDependencies(buf.Bytes())
+ deps := LoadDependencies(buf.Bytes(), config)
return deps, nil
}
@@ -146,24 +146,20 @@ func (resolver *MavenPomResolver) LoadDependencies() ([]*Dependency, error) {
func (resolver *MavenPomResolver) ResolveDependencies(deps []*Dependency, config *ConfigDeps, report *Report) error {
for _, dep := range deps {
func() {
- for _, l := range config.Licenses {
- for _, version := range strings.Split(l.Version, ",") {
- if l.Name == fmt.Sprintf("%s:%s", strings.Join(dep.GroupID, "."), dep.ArtifactID) && version == dep.Version {
- report.Resolve(&Result{
- Dependency: dep.String(),
- LicenseSpdxID: l.License,
- Version: dep.Version,
- })
- return
- }
- }
+ if license, ok := config.GetUserConfiguredLicense(dep.Name(), dep.Version); ok {
+ report.Resolve(&Result{
+ Dependency: dep.Name(),
+ LicenseSpdxID: license,
+ Version: dep.Version,
+ })
+ return
}
state := NotFound
err := resolver.ResolveLicense(config, &state, dep, report)
if err != nil {
- logger.Log.Warnf("Failed to resolve the license of <%s>: %v\n", dep.String(), state.String())
+ logger.Log.Warnf("Failed to resolve the license of <%s>: %v\n", dep.Name(), state.String())
report.Skip(&Result{
- Dependency: dep.String(),
+ Dependency: dep.Name(),
LicenseSpdxID: Unknown,
Version: dep.Version,
})
@@ -177,7 +173,7 @@ func (resolver *MavenPomResolver) ResolveDependencies(deps []*Dependency, config
func (resolver *MavenPomResolver) ResolveLicense(config *ConfigDeps, state *State, dep *Dependency, report *Report) error {
result1, err1 := resolver.ResolveJar(config, state, filepath.Join(resolver.repo, dep.Path(), dep.Jar()), dep.Version)
if result1 != nil {
- result1.Dependency = dep.String()
+ result1.Dependency = dep.Name()
report.Resolve(result1)
return nil
}
@@ -188,7 +184,7 @@ func (resolver *MavenPomResolver) ResolveLicense(config *ConfigDeps, state *Stat
return nil
}
- return fmt.Errorf("failed to resolve license for <%s> from jar or pom: %+v, %+v", dep.String(), err1, err2)
+ return fmt.Errorf("failed to resolve license for <%s> from jar or pom: %+v, %+v", dep.Name(), err1, err2)
}
// ResolveLicenseFromPom search for license in the pom file, which may appear in the header comments or in license element of xml
@@ -202,7 +198,7 @@ func (resolver *MavenPomResolver) ResolveLicenseFromPom(config *ConfigDeps, stat
if pom != nil && len(pom.Licenses) != 0 {
return &Result{
- Dependency: dep.String(),
+ Dependency: dep.Name(),
LicenseFilePath: pomFile,
LicenseContent: pom.Raw(),
LicenseSpdxID: pom.AllLicenses(config),
@@ -215,7 +211,7 @@ func (resolver *MavenPomResolver) ResolveLicenseFromPom(config *ConfigDeps, stat
return nil, err
} else if headerComments != "" {
*state |= FoundLicenseInPomHeader
- return resolver.IdentifyLicense(config, pomFile, dep.String(), headerComments, dep.Version)
+ return resolver.IdentifyLicense(config, pomFile, dep.Name(), headerComments, dep.Version)
}
return nil, fmt.Errorf("not found in pom file")
@@ -287,7 +283,7 @@ func SeemLicense(content string) bool {
return reMaybeLicense.MatchString(content)
}
-func LoadDependencies(data []byte) []*Dependency {
+func LoadDependencies(data []byte, config *ConfigDeps) []*Dependency {
depsTree := LoadDependenciesTree(data)
cnt := 0
@@ -299,6 +295,9 @@ func LoadDependencies(data []byte) []*Dependency {
queue := []*Dependency{}
for _, depTree := range depsTree {
+ if config.IsExcluded(depTree.Name(), depTree.Version) {
+ continue
+ }
queue = append(queue, depTree)
for len(queue) > 0 {
dep := queue[0]
@@ -326,9 +325,8 @@ func LoadDependenciesTree(data []byte) []*Dependency {
deps := make([]*Dependency, 0, len(rawDeps))
for _, rawDep := range rawDeps {
- gid := strings.Split(string(rawDep[reFind.SubexpIndex("gid")]), ".")
dep := &Dependency{
- GroupID: gid,
+ GroupID: string(rawDep[reFind.SubexpIndex("gid")]),
ArtifactID: string(rawDep[reFind.SubexpIndex("aid")]),
Packaging: string(rawDep[reFind.SubexpIndex("packaging")]),
Version: string(rawDep[reFind.SubexpIndex("version")]),
@@ -406,9 +404,8 @@ func (s *State) String() string {
}
type Dependency struct {
- GroupID []string
- ArtifactID, Version, Packaging, Scope string
- TransitiveDeps []*Dependency
+ GroupID, ArtifactID, Version, Packaging, Scope string
+ TransitiveDeps []*Dependency
}
func (dep *Dependency) Clone() *Dependency {
@@ -430,7 +427,7 @@ func (dep *Dependency) Count() int {
}
func (dep *Dependency) Path() string {
- return fmt.Sprintf("%v/%v/%v", strings.Join(dep.GroupID, "/"), dep.ArtifactID, dep.Version)
+ return fmt.Sprintf("%v/%v/%v", strings.ReplaceAll(dep.GroupID, ".", "/"), dep.ArtifactID, dep.Version)
}
func (dep *Dependency) Pom() string {
@@ -441,24 +438,8 @@ func (dep *Dependency) Jar() string {
return fmt.Sprintf("%v-%v.jar", dep.ArtifactID, dep.Version)
}
-func (dep *Dependency) String() string {
- buf := bytes.NewBuffer(nil)
- w := bufio.NewWriter(buf)
-
- _, err := w.WriteString(fmt.Sprintf("%v:%v", strings.Join(dep.GroupID, "."), dep.ArtifactID))
- if err != nil {
- logger.Log.Error(err)
- }
-
- for _, tDep := range dep.TransitiveDeps {
- _, err = w.WriteString(fmt.Sprintf("\n\t%v", tDep))
- if err != nil {
- logger.Log.Error(err)
- }
- }
-
- _ = w.Flush()
- return buf.String()
+func (dep *Dependency) Name() string {
+ return fmt.Sprintf("%v:%v", dep.GroupID, dep.ArtifactID)
}
// PomFile is used to extract license from the pom.xml file
diff --git a/pkg/deps/npm.go b/pkg/deps/npm.go
index 4cdb28f..ee47c27 100644
--- a/pkg/deps/npm.go
+++ b/pkg/deps/npm.go
@@ -190,7 +190,7 @@ func (resolver *NpmResolver) ResolvePackageLicense(pkgName, pkgPath string, conf
Dependency: pkgName,
}
// resolve from the package.json file
- if err := resolver.ResolvePkgFile(result, pkgPath, config.Licenses); err != nil {
+ if err := resolver.ResolvePkgFile(result, pkgPath, config); err != nil {
result.ResolveErrors = append(result.ResolveErrors, err)
}
@@ -203,7 +203,7 @@ func (resolver *NpmResolver) ResolvePackageLicense(pkgName, pkgPath string, conf
}
// ResolvePkgFile tries to find and parse the package.json file to capture the license field
-func (resolver *NpmResolver) ResolvePkgFile(result *Result, pkgPath string, licenses []*ConfigDepLicense) error {
+func (resolver *NpmResolver) ResolvePkgFile(result *Result, pkgPath string, config *ConfigDeps) error {
expectedPkgFile := filepath.Join(pkgPath, PkgFileName)
packageInfo, err := resolver.ParsePkgFile(expectedPkgFile)
if err != nil {
@@ -211,13 +211,9 @@ func (resolver *NpmResolver) ResolvePkgFile(result *Result, pkgPath string, lice
}
result.Version = packageInfo.Version
- for _, l := range licenses {
- for _, version := range strings.Split(l.Version, ",") {
- if l.Name == packageInfo.Name && version == packageInfo.Version {
- result.LicenseSpdxID = l.License
- return nil
- }
- }
+ if license, ok := config.GetUserConfiguredLicense(packageInfo.Name, packageInfo.Version); ok {
+ result.LicenseSpdxID = license
+ return nil
}
if lcs, ok := resolver.ResolveLicenseField(packageInfo.License); ok {
@@ -287,13 +283,9 @@ func (resolver *NpmResolver) ResolveLcsFile(result *Result, pkgPath string, conf
if result.LicenseSpdxID != "" {
return nil
}
- for _, l := range config.Licenses {
- for _, version := range strings.Split(l.Version, ",") {
- if l.Name == info.Name() && version == result.Version {
- result.LicenseSpdxID = l.License
- return nil
- }
- }
+ if license, ok := config.GetUserConfiguredLicense(info.Name(), result.Version); ok {
+ result.LicenseSpdxID = license
+ return nil
}
identifier, err := license.Identify(string(content), config.Threshold)
if err != nil {
diff --git a/pkg/deps/result.go b/pkg/deps/result.go
index 95a318b..66f87df 100644
--- a/pkg/deps/result.go
+++ b/pkg/deps/result.go
@@ -20,6 +20,7 @@ package deps
import (
"fmt"
"math"
+ "sort"
"strings"
)
@@ -56,6 +57,13 @@ func (report *Report) Skip(result *Result) {
}
func (report *Report) String() string {
+ sort.SliceStable(report.Resolved, func(i, j int) bool {
+ return report.Resolved[i].Dependency < report.Resolved[j].Dependency
+ })
+ sort.SliceStable(report.Skipped, func(i, j int) bool {
+ return report.Skipped[i].Dependency < report.Skipped[j].Dependency
+ })
+
dWidth, lWidth, vWidth := .0, .0, .0
for _, r := range report.Skipped {
dWidth = math.Max(float64(len(r.Dependency)), dWidth)