You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/04/16 20:35:58 UTC
[lucene] branch main updated: LUCENE-9931: Rename checksummed
regen. tasks FooInternal and generated wrappers Foo (#88)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new beafd11 LUCENE-9931: Rename checksummed regen. tasks FooInternal and generated wrappers Foo (#88)
beafd11 is described below
commit beafd113de187a3e58639460b87d22fb8d88b302
Author: Dawid Weiss <da...@carrotsearch.com>
AuthorDate: Fri Apr 16 22:35:51 2021 +0200
LUCENE-9931: Rename checksummed regen. tasks FooInternal and generated wrappers Foo (#88)
---
build.gradle | 4 ++--
gradle/documentation/markdown.gradle | 2 +-
gradle/generation/icu.gradle | 16 ++++++++--------
gradle/generation/javacc.gradle | 12 ++++++------
gradle/generation/jflex.gradle | 26 +++++++++++++-------------
gradle/generation/moman.gradle | 8 ++++----
gradle/generation/regenerate.gradle | 32 +++++++++++++++++++-------------
gradle/generation/snowball.gradle | 6 +++---
help/regeneration.txt | 30 +++++++++++++++---------------
9 files changed, 71 insertions(+), 65 deletions(-)
diff --git a/build.gradle b/build.gradle
index 4078bb7..c6cfc8d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -58,7 +58,7 @@ ext {
baseVersion = {
def m = (version =~ /^(\d+\.\d+\.\d+)(-(.+))?/)
if (!m) {
- throw GradleException("Can't strip version to just x.y.z: " + rootProject.version)
+ throw new GradleException("Can't strip version to just x.y.z: " + rootProject.version)
}
return m[0][1]
}()
@@ -66,7 +66,7 @@ ext {
majorVersion = {
def m = (version =~ /^(\d+)\.\d+\.\d+(-(.+))?/)
if (!m) {
- throw GradleException("Can't strip version to just major version: " + rootProject.version)
+ throw new GradleException("Can't strip version to just major version: " + rootProject.version)
}
return m[0][1] as int
}
diff --git a/gradle/documentation/markdown.gradle b/gradle/documentation/markdown.gradle
index 38a50ad..801c919 100644
--- a/gradle/documentation/markdown.gradle
+++ b/gradle/documentation/markdown.gradle
@@ -78,7 +78,7 @@ configure(project(':lucene:documentation')) {
def regex = ~/\bdefaultCodec\s*=\s*LOADER\s*\.\s*lookup\s*\(\s*"([^"]+)"\s*\)\s*;/
def matcher = regex.matcher(defaultCodecFile.getText('UTF-8'))
if (!matcher.find()) {
- throw GradleException("Cannot determine default codec from file ${defaultCodecFile}")
+ throw new GradleException("Cannot determine default codec from file ${defaultCodecFile}")
}
return matcher.group(1).toLowerCase(Locale.ROOT)
})
diff --git a/gradle/generation/icu.gradle b/gradle/generation/icu.gradle
index 112f2dc..3d36f94 100644
--- a/gradle/generation/icu.gradle
+++ b/gradle/generation/icu.gradle
@@ -69,7 +69,7 @@ configure(project(":lucene:analysis:icu")) {
def icuCompileTask = Os.isFamily(Os.FAMILY_WINDOWS) ? "compileIcuWindows" : "compileIcuLinux"
- task genUtr30DataFiles() {
+ task genUtr30DataFilesInternal() {
dependsOn icuCompileTask
// May be undefined yet, so use a provider.
@@ -119,7 +119,7 @@ configure(project(":lucene:analysis:icu")) {
}
}
- task genRbbi() {
+ task genRbbiInternal() {
// May be undefined yet, so use a provider.
dependsOn { sourceSets.tools.runtimeClasspath }
@@ -141,8 +141,8 @@ configure(project(":lucene:analysis:icu")) {
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(genUtr30DataFiles, [ ignoreWithSource: icuCompileTask ])
- regenerate.dependsOn wrapWithPersistentChecksums(genRbbi)
+ regenerate.dependsOn wrapWithPersistentChecksums(genUtr30DataFilesInternal, [ ignoreWithSource: icuCompileTask ])
+ regenerate.dependsOn wrapWithPersistentChecksums(genRbbiInternal)
task compileIcuWindows() {
doFirst {
@@ -236,7 +236,7 @@ configure(project(":lucene:analysis:icu")) {
// Regenerates UnicodeProps.java
configure(project(":lucene:analysis:common")) {
- task generateUnicodeProps() {
+ task generateUnicodePropsInternal() {
def icuConfig = rootProject.configurations.icu_68
def outputFile = file("src/java/org/apache/lucene/analysis/util/UnicodeProps.java")
@@ -260,13 +260,13 @@ configure(project(":lucene:analysis:common")) {
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(generateUnicodeProps, [ andThenTasks: "spotlessApply" ])
+ regenerate.dependsOn wrapWithPersistentChecksums(generateUnicodePropsInternal, [ andThenTasks: "spotlessApply" ])
}
// UnicodeEmojiProperties.jflex
configure(project(":lucene:core")) {
- task generateEmojiProperties() {
+ task generateEmojiPropertiesInternal() {
def icuConfig = rootProject.configurations.icu_62
def outputFile = file("src/data/jflex/UnicodeEmojiProperties.jflex")
@@ -290,5 +290,5 @@ configure(project(":lucene:core")) {
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(generateEmojiProperties)
+ regenerate.dependsOn wrapWithPersistentChecksums(generateEmojiPropertiesInternal)
}
\ No newline at end of file
diff --git a/gradle/generation/javacc.gradle b/gradle/generation/javacc.gradle
index 59bf460..bca31e4 100644
--- a/gradle/generation/javacc.gradle
+++ b/gradle/generation/javacc.gradle
@@ -107,7 +107,7 @@ def commonCleanups = { FileTree generatedFiles ->
}
configure(project(":lucene:queryparser")) {
- task javaccParserClassic(type: JavaCCTask) {
+ task javaccParserClassicInternal(type: JavaCCTask) {
description "Regenerate classic query parser from lucene/queryparser/classic/QueryParser.jj"
group "generation"
@@ -144,7 +144,7 @@ configure(project(":lucene:queryparser")) {
}
}
- task javaccParserSurround(type: JavaCCTask) {
+ task javaccParserSurroundInternal(type: JavaCCTask) {
description "Regenerate surround query parser from lucene/queryparser/surround/parser/QueryParser.jj"
group "generation"
@@ -169,7 +169,7 @@ configure(project(":lucene:queryparser")) {
}
}
- task javaccParserFlexible(type: JavaCCTask) {
+ task javaccParserFlexibleInternal(type: JavaCCTask) {
description "Regenerate flexible query parser from queryparser/flexible/standard/parser/StandardSyntaxParser.jj"
group "generation"
@@ -252,9 +252,9 @@ configure(project(":lucene:queryparser")) {
description "Regenerate query parsers (javacc syntax definitions)."
group "generation"
- dependsOn wrapWithPersistentChecksums(javaccParserClassic, [ andThenTasks: "spotlessApply" ]),
- wrapWithPersistentChecksums(javaccParserSurround, [ andThenTasks: "spotlessApply" ]),
- wrapWithPersistentChecksums(javaccParserFlexible, [ andThenTasks: "spotlessApply" ])
+ dependsOn wrapWithPersistentChecksums(javaccParserClassicInternal, [ andThenTasks: "spotlessApply" ]),
+ wrapWithPersistentChecksums(javaccParserSurroundInternal, [ andThenTasks: "spotlessApply" ]),
+ wrapWithPersistentChecksums(javaccParserFlexibleInternal, [ andThenTasks: "spotlessApply" ])
}
regenerate.dependsOn javacc
diff --git a/gradle/generation/jflex.gradle b/gradle/generation/jflex.gradle
index 66b0039..6bb5804 100644
--- a/gradle/generation/jflex.gradle
+++ b/gradle/generation/jflex.gradle
@@ -32,7 +32,7 @@ def skeletonDefault = file("${resources}/skeleton.default.txt")
def skeletonNoBufferExpansion = file("${resources}/skeleton.disable.buffer.expansion.txt")
configure(project(":lucene:core")) {
- task generateStandardTokenizer(type: JFlexTask) {
+ task generateStandardTokenizerInternal(type: JFlexTask) {
description "Regenerate StandardTokenizerImpl.java"
group "generation"
@@ -49,11 +49,11 @@ configure(project(":lucene:core")) {
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(generateStandardTokenizer, [ andThenTasks: "spotlessApply" ])
+ regenerate.dependsOn wrapWithPersistentChecksums(generateStandardTokenizerInternal, [ andThenTasks: "spotlessApply" ])
}
configure(project(":lucene:analysis:common")) {
- task generateTlds() {
+ task generateTldsInternal() {
def tldZones = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
def jflexMacro = file("src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex")
def tldList = file("src/test/org/apache/lucene/analysis/email/TLDs.txt")
@@ -82,7 +82,7 @@ configure(project(":lucene:analysis:common")) {
}
}
- task generateWikipediaTokenizer(type: JFlexTask) {
+ task generateWikipediaTokenizerInternal(type: JFlexTask) {
description "Regenerate WikipediaTokenizerImpl.java"
group "generation"
@@ -90,7 +90,7 @@ configure(project(":lucene:analysis:common")) {
skeleton = skeletonDefault
}
- task generateClassicTokenizer(type: JFlexTask) {
+ task generateClassicTokenizerInternal(type: JFlexTask) {
description "Regenerate ClassicTokenizerImpl.java"
group "generation"
@@ -98,7 +98,7 @@ configure(project(":lucene:analysis:common")) {
skeleton = skeletonDefault
}
- task generateUAX29URLEmailTokenizer(type: JFlexTask) {
+ task generateUAX29URLEmailTokenizerInternal(type: JFlexTask) {
description "Regenerate UAX29URLEmailTokenizerImpl.java"
group "generation"
@@ -108,7 +108,7 @@ configure(project(":lucene:analysis:common")) {
// Don't enforce strict dependency (although it would make sense to regenerate both).
// Just ensure we run after TLD regeneration, if it runs too.
- mustRunAfter generateTlds
+ mustRunAfter generateTldsInternal
// Additional input (included by the source jflex file)
inputs.file file('src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex')
@@ -127,7 +127,7 @@ configure(project(":lucene:analysis:common")) {
}
}
- task generateHTMLStripCharFilter(type: JFlexTask) {
+ task generateHTMLStripCharFilterInternal(type: JFlexTask) {
description "Regenerate HTMLStripCharFilter.java"
group "generation"
@@ -154,11 +154,11 @@ configure(project(":lucene:analysis:common")) {
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(generateWikipediaTokenizer, [ andThenTasks: "spotlessApply" ]),
- wrapWithPersistentChecksums(generateClassicTokenizer, [ andThenTasks: "spotlessApply" ]),
- wrapWithPersistentChecksums(generateUAX29URLEmailTokenizer, [ andThenTasks: "spotlessApply" ]),
- wrapWithPersistentChecksums(generateHTMLStripCharFilter, [ andThenTasks: "spotlessApply" ]),
- wrapWithPersistentChecksums(generateTlds, [ andThenTasks: "spotlessApply" ])
+ regenerate.dependsOn wrapWithPersistentChecksums(generateWikipediaTokenizerInternal, [ andThenTasks: "spotlessApply" ]),
+ wrapWithPersistentChecksums(generateClassicTokenizerInternal, [ andThenTasks: "spotlessApply" ]),
+ wrapWithPersistentChecksums(generateUAX29URLEmailTokenizerInternal, [ andThenTasks: "spotlessApply" ]),
+ wrapWithPersistentChecksums(generateHTMLStripCharFilterInternal, [ andThenTasks: "spotlessApply" ]),
+ wrapWithPersistentChecksums(generateTldsInternal, [ andThenTasks: "spotlessApply" ])
}
class JFlexTask extends DefaultTask {
diff --git a/gradle/generation/moman.gradle b/gradle/generation/moman.gradle
index 39b6e87..be58853 100644
--- a/gradle/generation/moman.gradle
+++ b/gradle/generation/moman.gradle
@@ -38,7 +38,7 @@ configure(project(":lucene:core")) {
}
}
- task utilGenPacked(dependsOn: installMoman) {
+ task utilGenPackedInternal(dependsOn: installMoman) {
def targetDir = file("src/java/org/apache/lucene/util/packed")
inputs.property("source", momanSource)
@@ -66,7 +66,7 @@ configure(project(":lucene:core")) {
}
}
- task utilGenLev(dependsOn: installMoman) {
+ task utilGenLevInternal(dependsOn: installMoman) {
def targetDir = file("src/java/org/apache/lucene/util/automaton")
inputs.property("source", momanSource)
@@ -99,8 +99,8 @@ configure(project(":lucene:core")) {
description "Regenerate Moman-based sources."
group "generation"
- dependsOn wrapWithPersistentChecksums(utilGenPacked, [ andThenTasks: "spotlessApply" ])
- dependsOn wrapWithPersistentChecksums(utilGenLev, [ andThenTasks: "spotlessApply" ])
+ dependsOn wrapWithPersistentChecksums(utilGenPackedInternal, [ andThenTasks: "spotlessApply" ])
+ dependsOn wrapWithPersistentChecksums(utilGenLevInternal, [ andThenTasks: "spotlessApply" ])
}
regenerate.dependsOn moman
diff --git a/gradle/generation/regenerate.gradle b/gradle/generation/regenerate.gradle
index 14ebc65..773f976 100644
--- a/gradle/generation/regenerate.gradle
+++ b/gradle/generation/regenerate.gradle
@@ -38,18 +38,18 @@ configure([
// This utility method implements the logic required for "persistent" incremental
// source-generating tasks. The idea is simple, the implementation quite complex.
//
- // The idea is that, given source-generating task "sourceTask" we create
- // a bunch of other tasks that perform checksum generation, validation and sourceTask
- // skipping; example:
+ // The idea is that, given source-generating task "sourceTaskInternal" (note the suffix),
+ // we create a bunch of other tasks that perform checksum generation, validation and
+ // source task skipping; example (sourceTask has Internal suffix stripped)
//
// ${sourceTask}ChecksumLoad
// ${sourceTask}ChecksumSave
// ${sourceTask}ChecksumCheck (fails if checksums are inconsistent)
- // maybe${sourceTask} dependsOn [checksum-load, sourceTask, checksum-save]
+ // ${sourceTask} dependsOn [checksum-load, ${sourceTask}Internal, checksum-save]
//
// Checksums are persisted and computed from sourceTask's inputs/outputs. If the
- // persisted checksums are identical to current checksums, sourceTask
- // is skipped (via sourceTask.onlyIf { false }).
+ // persisted checksums are identical to current checksums, source task
+ // is skipped (via sourceTaskInternal.onlyIf { false }).
//
// Implementation-wise things get complicated because gradle doesn't have the notion
// of "ordered" task execution with respect to task AND its dependencies (we can add
@@ -60,6 +60,12 @@ configure([
// andThenTasks: other tasks that should be scheduled to run after source task and
// before checksum calculation.
wrapWithPersistentChecksums = { Task sourceTask, Map<String, Object> extraConfig = [:] ->
+ if (!sourceTask.name.endsWith("Internal")) {
+ throw new GradleException("Wrapped task must follow the convention name of *Internal: ${sourceTask.name}")
+ }
+
+ String sourceTaskName = sourceTask.name.replaceAll('Internal$', '')
+
def toList = { value ->
if (value instanceof List) {
return value
@@ -74,7 +80,7 @@ configure([
List<Object> ignoreWithSource = toList(extraConfig.get("ignoreWithSource"))
// Create checksum-loader task.
- Task checksumLoadTask = tasks.create("${sourceTask.name}ChecksumLoad", {
+ Task checksumLoadTask = tasks.create("${sourceTaskName}ChecksumLoad", {
ext {
checksumMatch = true
}
@@ -94,7 +100,7 @@ configure([
ext.actualChecksums = actualChecksums
// Load any previously written checksums
- ext.checksumsFile = project.file("src/generated/checksums/${sourceTask.name}.json")
+ ext.checksumsFile = project.file("src/generated/checksums/${sourceTaskName}.json")
Map<String, String> savedChecksums = [:]
if (checksumsFile.exists()) {
savedChecksums = new JsonSlurper().parse(checksumsFile) as Map
@@ -105,7 +111,7 @@ configure([
}
})
- Task checksumCheckTask = tasks.create("${sourceTask.name}ChecksumCheck", {
+ Task checksumCheckTask = tasks.create("${sourceTaskName}ChecksumCheck", {
dependsOn checksumLoadTask
doFirst {
@@ -119,7 +125,7 @@ configure([
expected = expected - same
throw new GradleException("Checksums mismatch for derived resources; you might have" +
- " modified a generated resource (regenerate task: ${sourceTask.path}IfChanged):\n" +
+ " modified a generated resource (regenerate task: ${sourceTask.name}):\n" +
"Actual:\n ${actual.entrySet().join('\n ')}\n\n" +
"Expected:\n ${expected.entrySet().join('\n ')}"
)
@@ -128,7 +134,7 @@ configure([
})
check.dependsOn checksumCheckTask
- Task checksumSaveTask = tasks.create("${sourceTask.name}ChecksumSave", {
+ Task checksumSaveTask = tasks.create("${sourceTaskName}ChecksumSave", {
dependsOn checksumLoadTask
doFirst {
@@ -150,7 +156,7 @@ configure([
}
})
- Task conditionalTask = tasks.create("${sourceTask.name}IfChanged", {
+ Task conditionalTask = tasks.create("${sourceTaskName}", {
def deps = [
checksumLoadTask,
sourceTask,
@@ -162,7 +168,7 @@ configure([
mustRunInOrder deps
doFirst {
- if (checksumLoadTask.checksumMatch) {
+ if (checksumLoadTask.checksumMatch && !sourceTask.didWork) {
logger.lifecycle("Checksums consistent with sources, skipping task: ${sourceTask.path}")
}
}
diff --git a/gradle/generation/snowball.gradle b/gradle/generation/snowball.gradle
index 1404cc7..5e8423b 100644
--- a/gradle/generation/snowball.gradle
+++ b/gradle/generation/snowball.gradle
@@ -92,7 +92,7 @@ configure(project(":lucene:analysis:common")) {
}
// runs shell script to regenerate stemmers, base stemming subclasses, test data, and stopwords.
- task snowball() {
+ task snowballInternal() {
description "Regenerate snowball stemmers."
group "generation"
@@ -117,7 +117,7 @@ configure(project(":lucene:analysis:common")) {
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
doFirst {
// Just emit a big fat error message. Fail the build so that checksums are not regenerated.
- throw GradleException("Snowball generation does not work on Windows (patch and bash must be available).")
+ throw new GradleException("Snowball generation does not work on Windows (patch and bash must be available).")
}
} else {
dependsOn downloadSnowballStemmers
@@ -133,5 +133,5 @@ configure(project(":lucene:analysis:common")) {
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(snowball, [ andThenTasks: "spotlessApply", ignoreWithSource: [downloadSnowballStemmers, downloadSnowballWebsite, downloadSnowballData] ])
+ regenerate.dependsOn wrapWithPersistentChecksums(snowballInternal, [ andThenTasks: "spotlessApply", ignoreWithSource: [downloadSnowballStemmers, downloadSnowballWebsite, downloadSnowballData] ])
}
diff --git a/help/regeneration.txt b/help/regeneration.txt
index a9cd170..cc51ae4 100644
--- a/help/regeneration.txt
+++ b/help/regeneration.txt
@@ -77,15 +77,16 @@ gradlew -p lucene/analysis/common tasks --group generation
which displays (at the moment of writing):
-generateClassicTokenizerIfChanged - Regenerate ClassicTokenizerImpl.java (if sources changed)
-generateHTMLStripCharFilterIfChanged - Regenerate HTMLStripCharFilter.java (if sources changed)
-generateTldsIfChanged - Regenerate top-level domain jflex macros and tests (if sources changed)
-generateUAX29URLEmailTokenizerIfChanged - Regenerate UAX29URLEmailTokenizerImpl.java (if sources changed)
-generateWikipediaTokenizerIfChanged - Regenerate WikipediaTokenizerImpl.java (if sources changed)
+generateClassicTokenizer - Regenerate ClassicTokenizerImpl.java (if sources changed)
+generateHTMLStripCharFilter - Regenerate HTMLStripCharFilter.java (if sources changed)
+generateTlds - Regenerate top-level domain jflex macros and tests (if sources changed)
+generateUAX29URLEmailTokenizer - Regenerate UAX29URLEmailTokenizerImpl.java (if sources changed)
+generateWikipediaTokenizer - Regenerate WikipediaTokenizerImpl.java (if sources changed)
regenerate - Rerun any code or static data generation tasks.
snowball - Regenerates snowball stemmers.
-You may wonder what all those *IfChanged tasks are...
+You may wonder why none of these tasks actually exist in gradle source files (identically
+named tasks with a suffix "Internal" exist).
Resource checksums, incremental generation and advanced topics
@@ -101,14 +102,13 @@ plain console, for example:
gradlew -p lucene/analysis/common regenerate --console=plain
...
-> Task :lucene:analysis:common:generateUnicodePropsIfChanged
-Checksums consistent with sources, skipping task: :lucene:analysis:common:generateUnicodeProps
+> Task :lucene:analysis:common:generateUnicodeProps
+Checksums consistent with sources, skipping task: :lucene:analysis:common:generateUnicodePropsInternal
...
-This shouldn't worry you at all. The "*IfChanged" tasks wrap the actual generation
-tasks and verify whether the inputs and outputs of a task have changed. If so, the task
-is run (and follow-up task such as tidy are scheduled). If the checksums are identical to
-what was previously saved, the regeneration task is skipped.
+This shouldn't worry you at all - the internal tasks are skipped by wrappers
+if the inputs and outputs of the internal task have not changed. If they have changed,
+the task is re-run and followed up by other tasks, such as code-formatting (tidy).
Of course, sometimes you may want to *force* the regeneration task to run, even if the
checksums indicate nothing has changed. This may happen because of several reasons:
@@ -130,12 +130,12 @@ gradlew -p lucene/analysis/common regenerate --rerun-tasks
Scoping the call to a particular task will also work:
-gradlew -p lucene/analysis/common generateUnicodePropsIfChanged --rerun-tasks
+gradlew -p lucene/analysis/common generateUnicodeProps --rerun-tasks
You *should not* call the underlying generation task directly; this is possible
but discouraged:
-gradlew -p lucene/analysis/common generateUnicodeProps --rerun-tasks
+gradlew -p lucene/analysis/common generateUnicodePropsInternal --rerun-tasks
The reason is that some of these generation tasks require follow-up (for example
source code tidying) and, more importantly, the checksums for these
@@ -145,4 +145,4 @@ with checksum mismatches).
Finally, if you do feel like force-regenerating everything, remember to exclude this
monster...
-gradlew regenerate -x generateUAX29URLEmailTokenizer --rerun-tasks
+gradlew regenerate -x generateUAX29URLEmailTokenizerInternal --rerun-tasks