You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2025/09/18 17:16:18 UTC
(lucene) branch main updated: Use git-ls-files as the source of file paths for license checks. (#15200)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 5227fd72bd0 Use git-ls-files as the source of file paths for license checks. (#15200)
5227fd72bd0 is described below
commit 5227fd72bd0c65224eb643bc2dfbaf77dd90f045
Author: Dawid Weiss <da...@carrotsearch.com>
AuthorDate: Thu Sep 18 19:16:12 2025 +0200
Use git-ls-files as the source of file paths for license checks. (#15200)
---
.hgignore | 4 -
.../plugins/gitinfo/GitFileListValueSource.java | 84 +++++++++++
.../gradle/plugins/gitinfo/GitInfoExtension.java | 7 +
.../gradle/plugins/gitinfo/GitInfoPlugin.java | 83 ++++++-----
.../gradle/plugins/gitinfo/GitInfoValueSource.java | 15 +-
...xtension.java => GitValueSourceParameters.java} | 26 ++--
.../plugins/licenses/CheckLicensesPlugin.java | 156 ++++++++++++---------
build.gradle | 1 +
8 files changed, 248 insertions(+), 128 deletions(-)
diff --git a/.hgignore b/.hgignore
deleted file mode 100644
index 1e17ec787de..00000000000
--- a/.hgignore
+++ /dev/null
@@ -1,4 +0,0 @@
-syntax: glob
-*/build/*
-*.class
-
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitFileListValueSource.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitFileListValueSource.java
new file mode 100644
index 00000000000..d00a28afa73
--- /dev/null
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitFileListValueSource.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.gradle.plugins.gitinfo;
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import javax.inject.Inject;
+import org.gradle.api.GradleException;
+import org.gradle.api.provider.ValueSource;
+import org.gradle.process.ExecOperations;
+import org.slf4j.LoggerFactory;
+
+/** Returns a list of versioned and non-versioned (but not ignored) git files. */
+public abstract class GitFileListValueSource
+ implements ValueSource<List<String>, GitValueSourceParameters> {
+
+ @Inject
+ public abstract ExecOperations getExecOps();
+
+ @Override
+ public List<String> obtain() {
+ try (var baos = new ByteArrayOutputStream();
+ var out = new BufferedOutputStream(baos)) {
+ var result =
+ getExecOps()
+ .exec(
+ spec -> {
+ spec.setStandardOutput(out);
+ spec.setErrorOutput(out);
+ spec.setIgnoreExitValue(true);
+
+ getParameters().configure(spec);
+
+ spec.setExecutable(getParameters().getGitExec().get());
+ spec.args(
+ "ls-files",
+ // show cached
+ "-c",
+ // show others
+ "-o",
+ // apply .gitignore exclusions
+ "--exclude-standard",
+ // don't quote paths, 0-terminate strings.
+ "-z");
+ });
+ out.flush();
+
+ // Assume the output is UTF-8, even if it has 0 eols.
+ String gitOutput = baos.toString(StandardCharsets.UTF_8);
+
+ if (result.getExitValue() != 0) {
+ // Something went wrong. Assume this isn't a git checkout and
+ // return an empty list of files.
+ LoggerFactory.getLogger(getClass())
+ .warn(
+ "Failed executing git (exit status: {}). An empty list of versioned files will be used (run 'git init'?).",
+ result.getExitValue());
+
+ return List.of();
+ }
+
+ return List.of(gitOutput.split("\u0000"));
+ } catch (IOException e) {
+ throw new GradleException("Errors calling git to fetch local repository status.", e);
+ }
+ }
+}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.java
index f6b3df2ca33..84832c9530c 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.java
@@ -17,6 +17,7 @@
package org.apache.lucene.gradle.plugins.gitinfo;
import org.gradle.api.file.FileSystemLocation;
+import org.gradle.api.provider.ListProperty;
import org.gradle.api.provider.MapProperty;
import org.gradle.api.provider.Property;
@@ -32,4 +33,10 @@ public abstract class GitInfoExtension {
* @return Return the location of {@code .git} directory (or file, if worktrees are used).
*/
public abstract Property<FileSystemLocation> getDotGitDir();
+
+ /**
+ * @return Return a set of all versioned and non-versioned files (that are not ignored by {@code
+ * .gitignore}).
+ */
+ public abstract ListProperty<String> getAllNonIgnoredProjectFiles();
}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoPlugin.java
index a7f64593d77..e38a8d72df4 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoPlugin.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoPlugin.java
@@ -19,55 +19,68 @@ package org.apache.lucene.gradle.plugins.gitinfo;
import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;
+import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension;
+import org.gradle.api.Action;
import org.gradle.api.GradleException;
import org.gradle.api.Project;
import org.gradle.api.file.Directory;
+import org.gradle.api.file.FileSystemLocation;
+import org.gradle.api.provider.Property;
+import org.gradle.api.provider.ValueSourceSpec;
public class GitInfoPlugin extends LuceneGradlePlugin {
@Override
public void apply(Project project) {
applicableToRootProjectOnly(project);
- var gitInfoProvider =
- project
- .getProviders()
- .of(
- GitInfoValueSource.class,
- spec -> {
- spec.getParameters().getRootProjectDir().set(project.getProjectDir());
- });
-
var gitInfoExtension =
project.getExtensions().create(GitInfoExtension.NAME, GitInfoExtension.class);
+ var providers = project.getProviders();
- gitInfoExtension.getGitInfo().value(gitInfoProvider).finalizeValueOnRead();
-
- gitInfoExtension
- .getDotGitDir()
+ Property<FileSystemLocation> dotGitDir = gitInfoExtension.getDotGitDir();
+ dotGitDir
.convention(
- project
- .getProviders()
- .provider(
- () -> {
- Directory projectDirectory =
- project.getRootProject().getLayout().getProjectDirectory();
- Path gitLocation = projectDirectory.getAsFile().toPath().resolve(".git");
- if (!Files.exists(gitLocation)) {
- // don't return anything from the provider if we can't locate the .git
- // folder. This will result in the property returning false from isPresent.
- return null;
- }
+ providers.provider(
+ () -> {
+ Directory projectDirectory =
+ project.getRootProject().getLayout().getProjectDirectory();
+ Path gitLocation = projectDirectory.getAsFile().toPath().resolve(".git");
+ if (!Files.exists(gitLocation)) {
+ // don't return anything from the provider if we can't locate the .git
+ // folder. This will result in the property returning false from isPresent.
+ return null;
+ }
+
+ if (Files.isDirectory(gitLocation)) {
+ return projectDirectory.dir(".git");
+ } else if (Files.isRegularFile(gitLocation)) {
+ return projectDirectory.file(".git");
+ } else {
+ throw new GradleException(
+ "Panic, .git location not a directory or file: "
+ + gitLocation.toAbsolutePath());
+ }
+ }))
+ .finalizeValueOnRead();
- if (Files.isDirectory(gitLocation)) {
- return projectDirectory.dir(".git");
- } else if (Files.isRegularFile(gitLocation)) {
- return projectDirectory.file(".git");
- } else {
- throw new GradleException(
- "Panic, .git location not a directory or file: "
- + gitLocation.toAbsolutePath());
- }
- }))
+ var gitExec =
+ project.getExtensions().getByType(LuceneBuildGlobalsExtension.class).externalTool("git");
+
+ Action<ValueSourceSpec<GitValueSourceParameters>> configureGitParams =
+ spec -> {
+ var params = spec.getParameters();
+ params.getRootProjectDir().set(project.getProjectDir());
+ params.getGitExec().set(gitExec);
+ params.getDotDir().set(dotGitDir);
+ };
+
+ gitInfoExtension
+ .getGitInfo()
+ .value(providers.of(GitInfoValueSource.class, configureGitParams))
.finalizeValueOnRead();
+
+ gitInfoExtension
+ .getAllNonIgnoredProjectFiles()
+ .value(providers.of(GitFileListValueSource.class, configureGitParams));
}
}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoValueSource.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoValueSource.java
index cd1045765b6..0f1cae5a2c7 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoValueSource.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoValueSource.java
@@ -25,19 +25,12 @@ import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.inject.Inject;
-import org.apache.tools.ant.taskdefs.condition.Os;
import org.gradle.api.GradleException;
-import org.gradle.api.file.DirectoryProperty;
import org.gradle.api.provider.ValueSource;
-import org.gradle.api.provider.ValueSourceParameters;
-import org.gradle.api.services.BuildServiceParameters;
import org.gradle.process.ExecOperations;
public abstract class GitInfoValueSource
- implements ValueSource<Map<String, String>, GitInfoValueSource.Parameters> {
- public abstract static class Parameters implements BuildServiceParameters, ValueSourceParameters {
- public abstract DirectoryProperty getRootProjectDir();
- }
+ implements ValueSource<Map<String, String>, GitValueSourceParameters> {
@Inject
public abstract ExecOperations getExecOps();
@@ -54,9 +47,9 @@ public abstract class GitInfoValueSource
spec.setErrorOutput(out);
spec.setIgnoreExitValue(true);
- spec.setWorkingDir(
- getParameters().getRootProjectDir().getAsFile().get().getAbsolutePath());
- spec.setExecutable(Os.isFamily(Os.FAMILY_WINDOWS) ? "git.exe" : "git");
+ getParameters().configure(spec);
+
+ spec.setExecutable(getParameters().getGitExec().get());
spec.args("status", "--porcelain=v2", "--branch");
});
out.flush();
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitValueSourceParameters.java
similarity index 57%
copy from build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.java
copy to build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitValueSourceParameters.java
index f6b3df2ca33..49b57265f8e 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitValueSourceParameters.java
@@ -16,20 +16,24 @@
*/
package org.apache.lucene.gradle.plugins.gitinfo;
+import org.gradle.api.file.DirectoryProperty;
import org.gradle.api.file.FileSystemLocation;
-import org.gradle.api.provider.MapProperty;
import org.gradle.api.provider.Property;
+import org.gradle.api.provider.ValueSourceParameters;
+import org.gradle.api.services.BuildServiceParameters;
+import org.gradle.process.ExecSpec;
-public abstract class GitInfoExtension {
- public static final String NAME = "gitinfo";
+public abstract class GitValueSourceParameters
+ implements BuildServiceParameters, ValueSourceParameters {
+ public abstract DirectoryProperty getRootProjectDir();
- /**
- * @return Return a map of key-value pairs extracted from the current git status.
- */
- public abstract MapProperty<String, String> getGitInfo();
+ public abstract Property<String> getGitExec();
- /**
- * @return Return the location of {@code .git} directory (or file, if worktrees are used).
- */
- public abstract Property<FileSystemLocation> getDotGitDir();
+ public abstract Property<FileSystemLocation> getDotDir();
+
+ void configure(ExecSpec spec) {
+ String workDir = getRootProjectDir().getAsFile().get().getAbsolutePath();
+ spec.setWorkingDir(workDir);
+ spec.environment("GIT_DIR", getDotDir().get().getAsFile().getAbsolutePath());
+ }
}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.java
index 628c4904cda..224e343ab89 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.java
@@ -16,8 +16,13 @@
*/
package org.apache.lucene.gradle.plugins.licenses;
+import java.io.File;
+import java.util.List;
import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;
+import org.apache.lucene.gradle.plugins.gitinfo.GitInfoExtension;
+import org.apache.tools.ant.types.selectors.SelectorUtils;
import org.gradle.api.Project;
+import org.gradle.api.specs.Spec;
/** This configures ASL and other license checks. */
public class CheckLicensesPlugin extends LuceneGradlePlugin {
@@ -48,79 +53,96 @@ public class CheckLicensesPlugin extends LuceneGradlePlugin {
private void configureCheckLicenses(CheckLicensesTask task) {
Project project = task.getProject();
- task.getReportFile().set(project.getLayout().getBuildDirectory().file("licenses-report.txt"));
- task.getFiles()
- .from(
- project.fileTree(
- ".",
- tree -> {
- // Exclude build outputs, ide files, .git.
- tree.exclude(".git");
- tree.exclude(".idea");
- tree.exclude(".muse");
- tree.exclude("**/build/**");
- tree.exclude("**/.gradle");
+ assert project.getRootProject() == project;
+ var allNonIgnoredFiles =
+ project.getExtensions().getByType(GitInfoExtension.class).getAllNonIgnoredProjectFiles();
- // Exclude generated stuff.
- tree.exclude("**/src/generated/**");
+ task.getReportFile().set(project.getLayout().getBuildDirectory().file("licenses-report.txt"));
- // Exclude github stuff (templates, workflows).
- tree.exclude(".github");
+ // Build a list of files excluded from the license check. Just reuse ant's glob patterns.
+ var rootDir = getProjectRootPath(project);
+ var excludedPaths =
+ List.of(
+ // Ignore binary files. Previously we used apache rat, which had a 'binary guesser'
+ // but it's faster to just exclude by name (rather than scan the file).
+ "**/*.adoc",
+ "**/*.bin",
+ "**/*.brk",
+ "**/*.bz2",
+ "**/*.dat",
+ "**/*.gif",
+ "**/*.gz",
+ "**/*.png",
+ "**/*.svg",
+ "**/*.xls",
+ "**/*.zip",
- // do not let RAT attempt to scan a python venv, it gets lost and confused...
- tree.exclude("**/.venv/**");
+ // Ignore build infrastructure and misc utility files.
+ ".asf.yaml",
+ ".dir-locals.el",
+ ".editorconfig",
+ ".git-blame-ignore-revs",
+ ".gitattributes",
+ ".github/**",
+ ".gitignore",
+ ".lift.toml",
+ ".vscode/**",
+ "LICENSE.txt",
+ "NOTICE.txt",
+ "build-options.properties",
+ "dev-tools/**",
+ "gradle/**",
+ "help/*.txt",
+ "lucene/licenses/*",
+ "versions.lock",
- // apache rat has a 'binary guesser'... I don't think this needs to be done at all
- // -
- // just exclude binaries here.
- tree.exclude("**/*.dat");
- tree.exclude("**/*.brk");
- tree.exclude("**/*.gz");
- tree.exclude("**/*.bin");
- tree.exclude("**/*.bz2");
- tree.exclude("**/*.gif");
- tree.exclude("**/*.svg");
- tree.exclude("lucene/analysis/smartcn/src/**/*.mem");
+ // Ignore resources in source folders, also generated resources.
+ "**/src/**/*.txt",
+ "**/src/**/*.properties",
+ "**/src/**/*.utf8",
+ "**/src/generated/**",
- // Only check these selected file patterns as folks have various .gitignore-d
- // resources generated by IDEs, etc.
- tree.include("**/*.gradle");
- tree.include("**/*.xml");
- tree.include("**/*.md");
- tree.include("**/*.py");
- tree.include("**/*.sh");
- tree.include("**/*.bat");
+ // Ignore other binary resources within sources. Try to be
+ // specific here.
+ "build-tools/build-infra-shadow/src/java/keep.me",
+ "lucene/CHANGES.txt",
+ "lucene/analysis.tests/src/**/*.aff",
+ "lucene/analysis.tests/src/**/*.dic",
+ "lucene/analysis/common/src/**/*.aff",
+ "lucene/analysis/common/src/**/*.dic",
+ "lucene/analysis/common/src/**/*.good",
+ "lucene/analysis/common/src/**/*.htm*",
+ "lucene/analysis/common/src/**/*.rslp",
+ "lucene/analysis/common/src/**/*.sug",
+ "lucene/analysis/common/src/**/*.wrong",
+ "lucene/analysis/icu/src/**/utr30.nrm",
+ "lucene/analysis/kuromoji/src/**/bocchan.utf-8",
+ "lucene/analysis/morfologik/src/**/*.dict",
+ "lucene/analysis/morfologik/src/**/*.info",
+ "lucene/analysis/morfologik/src/**/*.input",
+ "lucene/analysis/opennlp/src/**/en-test-lemmas.dict",
+ "lucene/analysis/smartcn/src/**/*.mem",
+ "lucene/analysis/stempel/src/**/*.tbl",
+ "lucene/benchmark/.gitignore",
+ "lucene/demo/src/**/knn-token-vectors",
+ "lucene/luke/src/**/ElegantIcons.ttf",
+ "lucene/test-framework/src/**/europarl.lines.txt.seek",
- // Include selected patterns from any source folders.
- tree.include("**/src/**");
- tree.exclude("**/src/**/*.png");
- tree.exclude("**/src/**/*.txt");
- tree.exclude("**/src/**/*.zip");
- tree.exclude("**/src/**/*.properties");
- tree.exclude("**/src/**/*.utf8");
+ // these may require a review, actually?
+ "lucene/queryparser/docs/**",
+ "lucene/benchmark/conf/*",
+ "lucene/benchmark/README.enwiki");
- // project-specific exclusions.
- tree.exclude("build-tools/build-infra-shadow/src/java/keep.me");
- tree.exclude("lucene/analysis/icu/src/**/utr30.nrm");
- tree.exclude("lucene/analysis/kuromoji/src/**/bocchan.utf-8");
- tree.exclude("lucene/analysis/morfologik/src/**/*.info");
- tree.exclude("lucene/analysis/morfologik/src/**/*.input");
- tree.exclude("lucene/analysis/morfologik/src/**/*.dict");
- tree.exclude("lucene/analysis/stempel/src/**/*.tbl");
- tree.exclude("lucene/analysis/opennlp/src/**/en-test-lemmas.dict");
- tree.exclude("lucene/demo/src/**/knn-token-vectors");
- tree.exclude("lucene/test-framework/src/**/europarl.lines.txt.seek");
- tree.exclude("lucene/analysis/common/src/**/*.aff");
- tree.exclude("lucene/analysis/common/src/**/*.dic");
- tree.exclude("lucene/analysis/common/src/**/*.good");
- tree.exclude("lucene/analysis/common/src/**/*.sug");
- tree.exclude("lucene/analysis/common/src/**/*.wrong");
- tree.exclude("lucene/analysis/common/src/**/*.rslp");
- tree.exclude("lucene/analysis/common/src/**/*.htm*");
- tree.exclude("lucene/analysis.tests/src/**/*.aff");
- tree.exclude("lucene/analysis.tests/src/**/*.dic");
- // Luke has an embedded ElegantIcons font (MIT licensed).
- tree.exclude("lucene/luke/src/**/ElegantIcons.ttf");
- }));
+ // I thought it'd be possible to somehow precompile those glob filters but apparently not.
+ // I guess it's fine if the list of patterns and files is of reasonable size.
+ Spec<File> maybeExcludeKnownExceptions =
+ file -> {
+ // relativize from root project directory. No need to replace path separators as ant is
+ // os-agnostic here.
+ var filePath = rootDir.relativize(file.toPath()).toString();
+ return excludedPaths.stream()
+ .noneMatch(pattern -> SelectorUtils.match(pattern, filePath));
+ };
+ task.getFiles().from(project.files(allNonIgnoredFiles).filter(maybeExcludeKnownExceptions));
}
}
diff --git a/build.gradle b/build.gradle
index 926c584bee7..b483f83f1db 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1,3 +1,4 @@
+import org.apache.lucene.gradle.plugins.gitinfo.GitInfoExtension
import org.apache.lucene.gradle.plugins.licenses.CheckLicensesTask
/*