You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2024/06/06 14:36:54 UTC
(lucene) branch branch_9x updated: Backport generation-compilation ordering fixes from #13282. Correct whitespace difference in opennlp-tools-1.9.4.jar.sha1.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new d089cd672be Backport generation-compilation ordering fixes from #13282. Correct whitespace difference in opennlp-tools-1.9.4.jar.sha1.
d089cd672be is described below
commit d089cd672be9145e4183413943922fd4d38bfc7a
Author: Dawid Weiss <da...@carrotsearch.com>
AuthorDate: Thu Jun 6 16:36:27 2024 +0200
Backport generation-compilation ordering fixes from #13282.
Correct whitespace difference in opennlp-tools-1.9.4.jar.sha1.
---
gradle/generation/antlr.gradle | 5 +++-
gradle/generation/forUtil.gradle | 15 +++++++++---
gradle/generation/icu.gradle | 14 +++++++++---
gradle/generation/javacc.gradle | 11 ++++++---
gradle/generation/jflex.gradle | 26 +++++++++++++++------
gradle/generation/moman.gradle | 9 ++++++--
gradle/generation/regenerate.gradle | 33 +++++++++++++++++++--------
gradle/generation/snowball.gradle | 6 ++++-
gradle/generation/unicode-test-classes.gradle | 11 ++++++---
help/regeneration.txt | 16 -------------
lucene/licenses/opennlp-tools-1.9.4.jar.sha1 | 2 +-
11 files changed, 99 insertions(+), 49 deletions(-)
diff --git a/gradle/generation/antlr.gradle b/gradle/generation/antlr.gradle
index 33992384fa1..9e5d3fcb7c7 100644
--- a/gradle/generation/antlr.gradle
+++ b/gradle/generation/antlr.gradle
@@ -82,7 +82,10 @@ configure(project(":lucene:expressions")) {
}
def generateAntlr = wrapWithPersistentChecksums(
- generateAntlrInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+ generateAntlrInternal, [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ])
regenerate.dependsOn generateAntlr
}
diff --git a/gradle/generation/forUtil.gradle b/gradle/generation/forUtil.gradle
index 96f5bfdf974..e1428234875 100644
--- a/gradle/generation/forUtil.gradle
+++ b/gradle/generation/forUtil.gradle
@@ -39,7 +39,10 @@ configure(project(":lucene:core")) {
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(generateForUtilInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+ regenerate.dependsOn wrapWithPersistentChecksums(generateForUtilInternal, [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ])
}
configure(project(":lucene:backward-codecs")) {
@@ -64,7 +67,10 @@ configure(project(":lucene:backward-codecs")) {
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil84Internal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+ regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil84Internal, [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ])
task generateForUtil90Internal() {
description "Regenerate gen_ForUtil.py"
@@ -86,6 +92,9 @@ configure(project(":lucene:backward-codecs")) {
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil90Internal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+ regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil90Internal, [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ])
}
diff --git a/gradle/generation/icu.gradle b/gradle/generation/icu.gradle
index 85d9084b1df..042f756bbdb 100644
--- a/gradle/generation/icu.gradle
+++ b/gradle/generation/icu.gradle
@@ -159,8 +159,13 @@ configure(project(":lucene:analysis:icu")) {
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(genUtr30DataFilesInternal, [ ignoreWithSource: icuCompileTask ])
- regenerate.dependsOn wrapWithPersistentChecksums(genRbbiInternal)
+ regenerate.dependsOn wrapWithPersistentChecksums(genUtr30DataFilesInternal, [
+ ignoreWithSource: icuCompileTask,
+ mustRunBefore: [ "compileJava" ]
+ ])
+ regenerate.dependsOn wrapWithPersistentChecksums(genRbbiInternal, [
+ mustRunBefore: [ "compileJava" ]
+ ])
task compileIcuWindows() {
doFirst {
@@ -281,5 +286,8 @@ configure(project(":lucene:analysis:common")) {
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(generateUnicodePropsInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+ regenerate.dependsOn wrapWithPersistentChecksums(generateUnicodePropsInternal, [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ])
}
diff --git a/gradle/generation/javacc.gradle b/gradle/generation/javacc.gradle
index 3d0dd916150..5409e3883a4 100644
--- a/gradle/generation/javacc.gradle
+++ b/gradle/generation/javacc.gradle
@@ -232,9 +232,14 @@ configure(project(":lucene:queryparser")) {
description "Regenerate query parsers (javacc syntax definitions)."
group "generation"
- dependsOn wrapWithPersistentChecksums(javaccParserClassicInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]),
- wrapWithPersistentChecksums(javaccParserSurroundInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]),
- wrapWithPersistentChecksums(javaccParserFlexibleInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+ def extraConfig = [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ]
+
+ dependsOn wrapWithPersistentChecksums(javaccParserClassicInternal, extraConfig),
+ wrapWithPersistentChecksums(javaccParserSurroundInternal, extraConfig),
+ wrapWithPersistentChecksums(javaccParserFlexibleInternal, extraConfig)
}
regenerate.dependsOn javacc
diff --git a/gradle/generation/jflex.gradle b/gradle/generation/jflex.gradle
index c533c3cfa4f..6c028b1279e 100644
--- a/gradle/generation/jflex.gradle
+++ b/gradle/generation/jflex.gradle
@@ -51,7 +51,10 @@ configure(project(":lucene:core")) {
}
}
- def generateStandardTokenizer = wrapWithPersistentChecksums(generateStandardTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+ def generateStandardTokenizer = wrapWithPersistentChecksums(generateStandardTokenizerInternal, [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ])
regenerate.dependsOn generateStandardTokenizer
}
@@ -149,7 +152,11 @@ configure(project(":lucene:analysis:common")) {
}
}
- def generateUAX29URLEmailTokenizer = wrapWithPersistentChecksums(generateUAX29URLEmailTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+ def generateUAX29URLEmailTokenizer = wrapWithPersistentChecksums(generateUAX29URLEmailTokenizerInternal, [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ])
+
// UAX29URLEmailTokenizerImpl.jflex includes: ASCIITLD.jflex
// so we make sure it is up to date. Also, make sure the ordering of internal
// tasks is enforced (just scheduling top-level wrappers doesn't mean their subtask graphs cannot be reordered).
@@ -197,7 +204,12 @@ configure(project(":lucene:analysis:common")) {
skeleton = skeletonDefault
}
- def generateHTMLStripCharFilter = wrapWithPersistentChecksums(generateHTMLStripCharFilterInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+ def extraConfig = [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ]
+
+ def generateHTMLStripCharFilter = wrapWithPersistentChecksums(generateHTMLStripCharFilterInternal, extraConfig)
// HTMLStripCharFilter.jflex includes HTMLCharacterEntities.jflex so we make sure it's up to date.
configure(generateHTMLStripCharFilter) {
dependsOn "generateHTMLCharacterEntities"
@@ -206,12 +218,12 @@ configure(project(":lucene:analysis:common")) {
dependsOn "generateHTMLCharacterEntitiesInternal"
}
- regenerate.dependsOn wrapWithPersistentChecksums(generateWikipediaTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]),
- wrapWithPersistentChecksums(generateClassicTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]),
+ regenerate.dependsOn wrapWithPersistentChecksums(generateWikipediaTokenizerInternal, extraConfig),
+ wrapWithPersistentChecksums(generateClassicTokenizerInternal, extraConfig),
generateUAX29URLEmailTokenizer,
- wrapWithPersistentChecksums(generateHTMLCharacterEntitiesInternal),
+ wrapWithPersistentChecksums(generateHTMLCharacterEntitiesInternal, extraConfig),
generateHTMLStripCharFilter,
- wrapWithPersistentChecksums(generateTldsInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+ wrapWithPersistentChecksums(generateTldsInternal, extraConfig)
}
class JFlexTask extends DefaultTask {
diff --git a/gradle/generation/moman.gradle b/gradle/generation/moman.gradle
index 62dbed15acf..f825090c27c 100644
--- a/gradle/generation/moman.gradle
+++ b/gradle/generation/moman.gradle
@@ -105,8 +105,13 @@ configure(project(":lucene:core")) {
description "Regenerate Moman-based sources."
group "generation"
- dependsOn wrapWithPersistentChecksums(utilGenPackedInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
- dependsOn wrapWithPersistentChecksums(utilGenLevInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+ def extraConfig = [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ]
+
+ dependsOn wrapWithPersistentChecksums(utilGenPackedInternal, extraConfig)
+ dependsOn wrapWithPersistentChecksums(utilGenLevInternal, extraConfig)
}
regenerate.dependsOn moman
diff --git a/gradle/generation/regenerate.gradle b/gradle/generation/regenerate.gradle
index da438e5e14e..06640a8619b 100644
--- a/gradle/generation/regenerate.gradle
+++ b/gradle/generation/regenerate.gradle
@@ -105,16 +105,21 @@ configure([
//
// The idea is that, given source-generating task "sourceTaskInternal" (note the suffix),
// we create a bunch of other tasks that perform checksum generation, validation and
- // source task skipping; example (sourceTask has Internal suffix stripped)
+ // source task skipping. For example, let's say we have a task 'genFooInternal";
+ // the following tasks would be created
//
- // ${sourceTask}ChecksumLoad
- // ${sourceTask}ChecksumSave
- // ${sourceTask}ChecksumCheck (fails if checksums are inconsistent)
- // ${sourceTask} dependsOn [checksum-load, ${sourceTask}Internal, checksum-save]
+ // genFooChecksumLoad
+ // genFooChecksumSave
+ // genFooChecksumCheck (fails if checksums are inconsistent)
+ //
+ // and the following set of dependencies would be created (with additional
+ // constraints to run them in this particular order!).
+ //
+ // genFoo.dependsOn [genFooChecksumLoad, genFooInternal, genFooChecksumSave]
//
// Checksums are persisted and computed from sourceTask's inputs/outputs. If the
- // persisted checksums are identical to current checksums, source task
- // is skipped (via sourceTaskInternal.onlyIf { false }).
+ // persisted checksums are identical to the now-current checksums, the "internal" task
+ // is skipped (using onlyIf { false }).
//
// Implementation-wise things get complicated because gradle doesn't have the notion
// of "ordered" task execution with respect to task AND its dependencies (we can add
@@ -122,8 +127,11 @@ configure([
//
// sourceTask - the task to wrap
// extraConfig - a map with extra (optional) configuration options.
- // andThenTasks: other tasks that should be scheduled to run after source task and
- // before checksum calculation.
+ // andThenTasks: any other tasks that should be scheduled to run after the internal task and
+ // before checksum calculation/ saving. spotless is a good example of this.
+ // ignoreWithSource: any other tasks that should be ignored if the internal task is ignored
+ // (checksums are identical)
+ // mustRunBefore: any tasks which should be scheduled to run after the internal task.
wrapWithPersistentChecksums = { Task sourceTask, Map<String, Object> extraConfig = [:] ->
if (!sourceTask.name.endsWith("Internal")) {
throw new GradleException("Wrapped task must follow the convention name of *Internal: ${sourceTask.name}")
@@ -144,6 +152,13 @@ configure([
List<Object> andThenTasks = toList(extraConfig.get("andThenTasks"))
List<Object> ignoreWithSource = toList(extraConfig.get("ignoreWithSource"))
+ // Schedule must-run-afters
+ List<Object> mustRunBefore = toList(extraConfig.get("mustRunBefore"))
+ // TODO: maybe ensure all task refs here are strings?
+ tasks.matching { it.name in mustRunBefore }.configureEach {
+ mustRunAfter sourceTask
+ }
+
// Create checksum-loader task.
Task checksumLoadTask = tasks.create("${sourceTaskName}ChecksumLoad", {
ext {
diff --git a/gradle/generation/snowball.gradle b/gradle/generation/snowball.gradle
index aa1e5fbccdf..378c621331e 100644
--- a/gradle/generation/snowball.gradle
+++ b/gradle/generation/snowball.gradle
@@ -133,5 +133,9 @@ configure(project(":lucene:analysis:common")) {
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(snowballInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"], ignoreWithSource: [downloadSnowballStemmers, downloadSnowballWebsite, downloadSnowballData] ])
+ regenerate.dependsOn wrapWithPersistentChecksums(snowballInternal, [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ ignoreWithSource: [downloadSnowballStemmers, downloadSnowballWebsite, downloadSnowballData],
+ mustRunBefore: [ "compileJava" ]
+ ])
}
diff --git a/gradle/generation/unicode-test-classes.gradle b/gradle/generation/unicode-test-classes.gradle
index cd2086edeb1..0bf25aa190d 100644
--- a/gradle/generation/unicode-test-classes.gradle
+++ b/gradle/generation/unicode-test-classes.gradle
@@ -43,7 +43,10 @@ configure(project(":lucene:test-framework")) {
}
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(generateEmojiTokenizationTestInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+ regenerate.dependsOn wrapWithPersistentChecksums(generateEmojiTokenizationTestInternal, [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ])
task generateJavaUnicodeWordBreakTestInternal() {
def unicodeVersion = "12.1"
@@ -71,6 +74,8 @@ configure(project(":lucene:test-framework")) {
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(generateJavaUnicodeWordBreakTestInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
-
+ regenerate.dependsOn wrapWithPersistentChecksums(generateJavaUnicodeWordBreakTestInternal, [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ])
}
diff --git a/help/regeneration.txt b/help/regeneration.txt
index 440acbf096d..dc7ced3b988 100644
--- a/help/regeneration.txt
+++ b/help/regeneration.txt
@@ -16,22 +16,6 @@ In many cases hitting (1) means you'll have to do (2) so let's discuss
these in order.
-SPECIAL NOTE
-------------
-
-Regeneration tasks currently don't play well with the current gradle version.
-To workaround the issue, compile everything first:
-
-gradlew compileJava
-
-Then run the commands as described in this document, excluding compileJava.
-For example to run "regenerate":
-
-gradlew regenerate -x compileJava
-
-More information: https://github.com/apache/lucene/issues/13240
-
-
Checksum validation errors
--------------------------
diff --git a/lucene/licenses/opennlp-tools-1.9.4.jar.sha1 b/lucene/licenses/opennlp-tools-1.9.4.jar.sha1
index b367a761a44..fa7e85dd48f 100644
--- a/lucene/licenses/opennlp-tools-1.9.4.jar.sha1
+++ b/lucene/licenses/opennlp-tools-1.9.4.jar.sha1
@@ -1 +1 @@
-dd5c4a6d82453bcccb78ba4ac90f166366dde12b
\ No newline at end of file
+dd5c4a6d82453bcccb78ba4ac90f166366dde12b