You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2025/10/28 17:18:06 UTC
(lucene) branch main updated: Final Gradle plugins to Java (#15377)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 912e9066398 Final Gradle plugins to Java (#15377)
912e9066398 is described below
commit 912e906639809c026bb17307cad766e5e2eb2571
Author: Dawid Weiss <da...@carrotsearch.com>
AuthorDate: Tue Oct 28 18:18:01 2025 +0100
Final Gradle plugins to Java (#15377)
* Partial, stopped when realizing gradle sucks.
* Rewrite RegenerateTasksSupportPlugin using after-projects-evaluated hook.
* Moved regeneration code into each subproject's build.gradle.
* Final cleanups.
* Don't complain about errorprone unless 'check' is mentioned explicitly.
* Cleanup deprecated APIs.
* Minor tweaks.
* Use a service instead of the deprecated hook.
* Move project reference outside of the task's execution code.
* Remove obsolete launch configurations.
* Remove exception from close.
---
build-tools/build-infra/build.gradle | 9 +-
.../src/main/groovy/regenerate.antlr.gradle | 106 -------
.../src/main/groovy/regenerate.forUtil.gradle | 78 -----
.../src/main/groovy/regenerate.icu.gradle | 310 -------------------
.../src/main/groovy/regenerate.javacc.gradle | 329 --------------------
.../src/main/groovy/regenerate.jflex.gradle | 275 ----------------
.../src/main/groovy/regenerate.kuromoji.gradle | 143 ---------
.../src/main/groovy/regenerate.moman.gradle | 114 -------
.../src/main/groovy/regenerate.nori.gradle | 90 ------
.../src/main/groovy/regenerate.snowball.gradle | 115 -------
.../groovy/regenerate.unicode-test-classes.gradle | 66 ----
.../gradle/plugins/hacks/WipeGradleTempPlugin.java | 159 ++++++----
.../gradle/plugins/java/ErrorPronePlugin.java | 15 +-
.../CheckJarChecksumsAndLicensesPlugin.java | 27 +-
.../gradle/plugins/licenses/CollectJarInfos.java | 5 +
...gin.java => LuceneRootConfigurationPlugin.java} | 6 +-
.../gradle/plugins/regenerate/JFlexTask.java | 125 ++++++++
.../gradle/plugins/regenerate/JavaCCTask.java | 160 ++++++++++
.../regenerate/RegenerateTasksSupportPlugin.java | 85 +++--
build.gradle | 14 -
gradle/ide/eclipse/run-solr-cloud.launch | 20 --
gradle/ide/eclipse/run-solr.launch | 20 --
gradle/ide/eclipse/run-test-cases.launch | 17 -
.../{moman => packed}/gen_BulkOperation.py | 0
.../{moman => packed}/gen_Packed64SingleBlock.py | 0
lucene/analysis/common/build.gradle | 103 +++++-
lucene/analysis/icu/build.gradle | 286 ++++++++++++++++-
lucene/analysis/kuromoji/build.gradle | 126 ++++++++
lucene/analysis/nori/build.gradle | 76 ++++-
lucene/benchmark-jmh/build.gradle | 4 +-
lucene/core/build.gradle | 344 +++++++++++++++++++++
.../checksums/regeneratePacked64SingleBlock.json | 4 +-
lucene/distribution.tests/build.gradle | 2 +-
lucene/expressions/build.gradle | 88 ++++++
lucene/queryparser/build.gradle | 215 ++++++++++++-
lucene/test-framework/build.gradle | 50 ++-
36 files changed, 1742 insertions(+), 1844 deletions(-)
diff --git a/build-tools/build-infra/build.gradle b/build-tools/build-infra/build.gradle
index 011c050c0cc..258cc71c8e1 100644
--- a/build-tools/build-infra/build.gradle
+++ b/build-tools/build-infra/build.gradle
@@ -17,8 +17,6 @@
plugins {
id "java-gradle-plugin"
- id "groovy-gradle-plugin"
-
alias(deps.plugins.forbiddenapis) apply false
}
@@ -40,14 +38,9 @@ gradlePlugin {
automatedPublishing = false
plugins {
- CheckEnvironmentPlugin {
- id = 'lucene.root-project.check-environment'
- implementationClass = 'org.apache.lucene.gradle.plugins.misc.CheckEnvironmentPlugin'
- }
-
RootProjectSetupPlugin {
id = 'lucene.root-project.setup'
- implementationClass = 'org.apache.lucene.gradle.plugins.misc.RootProjectSetupPlugin'
+ implementationClass = 'org.apache.lucene.gradle.plugins.misc.LuceneRootConfigurationPlugin'
}
}
}
diff --git a/build-tools/build-infra/src/main/groovy/regenerate.antlr.gradle b/build-tools/build-infra/src/main/groovy/regenerate.antlr.gradle
deleted file mode 100644
index cfcffadbb00..00000000000
--- a/build-tools/build-infra/src/main/groovy/regenerate.antlr.gradle
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-//
-// Regenerate antlr4 grammars.
-//
-
-configure(project(":lucene:expressions")) {
- configurations {
- antlr
- }
-
- dependencies {
- antlr deps.antlr.core
- }
-
- tasks.register("regenerateAntlr", {
- description = "Regenerate Javascript.g4"
-
- dependsOn configurations.antlr
-
- def generatedPatterns = [
- "**/JavascriptBaseVisitor.java",
- "**/JavascriptLexer.java",
- "**/JavascriptParser.java",
- "**/JavascriptVisitor.java",
- ]
-
- def sourceDir = "src/java/org/apache/lucene/expressions/js"
- def inputFiles = file("$sourceDir/Javascript.g4")
- def tempOutput = layout.buildDirectory.dir("antlr").get().asFile
- def outputFiles = fileTree(dir: sourceDir, includes: generatedPatterns)
-
- inputs.property "antlr-version", deps.antlr.core.get().version
- inputs.files inputFiles
- outputs.files outputFiles
-
- doFirst {
- buildGlobals.getFileOps().delete(tempOutput)
- buildGlobals.getExecOps().javaexec {
- mainClass = "org.antlr.v4.Tool"
- classpath = configurations.antlr
-
- ignoreExitValue = false
- args = [
- "-no-listener",
- "-visitor",
- "-package",
- "org.apache.lucene.expressions.js",
- "-o",
- tempOutput,
- inputFiles
- ]
- }
-
- def generatedFiles = fileTree(dir: tempOutput, includes: generatedPatterns)
- generatedFiles.each {file ->
- buildGlobals.modifyFile(file, {text ->
- text = text.replaceAll("public ((interface|class) Javascript\\w+)", "\$1")
- text = text.replaceAll("// Generated from .*",
- """
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// ANTLR GENERATED CODE: DO NOT EDIT.
-""")
- return text
- })
- }
-
- buildGlobals.getFileOps().copy({CopySpec spec ->
- spec.from(tempOutput)
- spec.into(sourceDir)
- spec.include(generatedPatterns)
- })
- }
- })
-}
diff --git a/build-tools/build-infra/src/main/groovy/regenerate.forUtil.gradle b/build-tools/build-infra/src/main/groovy/regenerate.forUtil.gradle
deleted file mode 100644
index 71d2a603a1d..00000000000
--- a/build-tools/build-infra/src/main/groovy/regenerate.forUtil.gradle
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.gradle.plugins.misc.QuietExec
-
-//
-// Regenerates "ForUtil.java" code used by codecs.
-//
-
-void addGenerateFromPythonTask(Project project, File genDir, String taskName, String pyScript, String javaFile) {
- project.getTasks().register(
- "regenerate" + taskName, QuietExec, {
- description = "Regenerate " + javaFile + " using " + pyScript + " in " + genDir
-
- def genScript = project.file("${genDir}/${pyScript}")
- def genOutput = project.file("${genDir}/${javaFile}")
- inputs.file genScript
- outputs.file genOutput
-
- workingDir = genDir
- executable = buildGlobals.externalTool("python3")
- args = ['-B', genScript.toString()]
- })
-}
-
-configure(project(":lucene:core")) {
- addGenerateFromPythonTask(project,
- file("src/java/org/apache/lucene/codecs/lucene104"), "ForUtil", "gen_ForUtil.py", "ForUtil.java")
-}
-
-configure(project(":lucene:backward-codecs")) {
- addGenerateFromPythonTask(project,
- file("src/java/org/apache/lucene/backward_codecs/lucene84"), "ForUtil84", "gen_ForUtil.py", "ForUtil.java")
-
- addGenerateFromPythonTask(project,
- file("src/java/org/apache/lucene/backward_codecs/lucene90"), "ForUtil90", "gen_ForUtil.py", "ForUtil.java")
-
- addGenerateFromPythonTask(project,
- file("src/java/org/apache/lucene/backward_codecs/lucene99"), "ForUtil99", "gen_ForUtil.py", "ForUtil.java")
-
- addGenerateFromPythonTask(project,
- file("src/java/org/apache/lucene/backward_codecs/lucene912"), "ForUtil912", "gen_ForUtil.py", "ForUtil.java")
- addGenerateFromPythonTask(project,
- file("src/java/org/apache/lucene/backward_codecs/lucene912"),
- "ForDeltaUtil912",
- "gen_ForDeltaUtil.py",
- "ForDeltaUtil.java")
-
- addGenerateFromPythonTask(project,
- file("src/java/org/apache/lucene/backward_codecs/lucene101"), "ForUtil101", "gen_ForUtil.py", "ForUtil.java")
- addGenerateFromPythonTask(project,
- file("src/java/org/apache/lucene/backward_codecs/lucene101"),
- "ForDeltaUtil101",
- "gen_ForDeltaUtil.py",
- "ForDeltaUtil.java")
-
- addGenerateFromPythonTask(project,
- file("src/java/org/apache/lucene/backward_codecs/lucene103"), "ForUtil103", "gen_ForUtil.py", "ForUtil.java")
- addGenerateFromPythonTask(project,
- file("src/java/org/apache/lucene/backward_codecs/lucene103"),
- "ForDeltaUtil103",
- "gen_ForDeltaUtil.py",
- "ForDeltaUtil.java")
-}
diff --git a/build-tools/build-infra/src/main/groovy/regenerate.icu.gradle b/build-tools/build-infra/src/main/groovy/regenerate.icu.gradle
deleted file mode 100644
index 63c5e3e5ea3..00000000000
--- a/build-tools/build-infra/src/main/groovy/regenerate.icu.gradle
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.tools.ant.taskdefs.condition.Os
-import java.nio.file.Files
-
-def resources = rootProject.file("gradle/regenerate/icu")
-
-/*
- * Regenerates ICU-related data files.
- *
- * This build file contains regeneration code utilizing both icu4j and icu4c.
- *
- * The icu4c version must match exactly the icu4j version in version.props:
- * The one on your system is probably different. This script will attempt to
- * download and compile a matching icu4c version automatically.
- */
-
-
-def setupIcuDependencies = { proj ->
- // Configure different icu4j dependencies.
- proj.configurations {
- icu_current
- groovy
- }
-
- proj.dependencies {
- icu_current deps.icu4j
- groovy deps.groovy
- }
-}
-
-def icu4jVersion = deps.icu4j.get().version
-
-configure(project(":lucene:analysis:icu")) {
- setupIcuDependencies(project)
- def utr30DataDir = file("src/data/utr30")
-
- def icuBuildDir = file("${buildDir}/icu")
- def icuBinDir
- def gennorm
- def icupkg
-
- if (Os.isFamily(Os.FAMILY_WINDOWS)) {
- icuBinDir = file("${icuBuildDir}/bin64")
- gennorm = file("${icuBinDir}/gennorm2.exe")
- icupkg = file("${icuBinDir}/icupkg.exe")
- } else {
- icuBinDir = file("${icuBuildDir}/icu/source/bin")
- gennorm = file("${icuBinDir}/gennorm2")
- icupkg = file("${icuBinDir}/icupkg")
- }
-
- def icuCompileTask = Os.isFamily(Os.FAMILY_WINDOWS) ? "compileIcuWindows" : "compileIcuLinux"
-
- tasks.register("regenerateUtr30DataFiles", {
- description = "Regenerate utf30 norm files using ICU"
-
- def icuConfig = configurations.icu_current
-
- dependsOn icuConfig
- dependsOn icuCompileTask
-
- regenerate {
- ifSkippedAlsoSkip = [icuCompileTask]
- }
-
- // May be undefined yet, so use a provider.
- dependsOn { sourceSets.tools.runtimeClasspath }
-
- // gennorm generates file order-dependent output, so make it constant here.
- def inputFiles = fileTree(dir: utr30DataDir, include: "*.txt")
- .asList().toSorted(Comparator.comparing { File f -> f.name })
- def outputFile = file("src/resources/org/apache/lucene/analysis/icu/utr30.nrm")
-
- inputs.files inputFiles
- inputs.property "icuConfig", icu4jVersion
- outputs.file outputFile
-
- doFirst {
- // all these steps must be done sequentially: it's a pipeline resulting in utr30.nrm
- buildGlobals.execOps.javaexec {
- mainClass = "org.apache.lucene.analysis.icu.GenerateUTR30DataFiles"
- classpath = sourceSets.tools.runtimeClasspath
-
- ignoreExitValue = false
- workingDir utr30DataDir
- args = [
- "release-${icu4jVersion.replace(".", "-")}"
- ]
- }
-
- buildGlobals.quietExec(it, {
- executable gennorm
- args = [
- "-v",
- "-s",
- utr30DataDir,
- "-o",
- "${buildDir}/utr30.tmp",
- *(inputFiles.collect { it.name })
- ]
- })
-
- buildGlobals.quietExec(it, {
- executable icupkg
- args = [
- "-tb",
- "${buildDir}/utr30.tmp",
- outputFile
- ]
- })
- }
- })
-
- tasks.register("regenerateRbbi", {
- description = "Regenerate rbbi rules using ICU"
-
- def icuConfig = configurations.icu_current
-
- dependsOn icuConfig
-
- // May be undefined yet, so use a provider.
- dependsOn { sourceSets.tools.runtimeClasspath }
-
- def sourceDir = file("src/data/uax29")
- def targetDir = file("src/resources/org/apache/lucene/analysis/icu/segmentation")
-
- inputs.files fileTree(dir: sourceDir, include: "*.rbbi")
- inputs.property "icuConfig", icu4jVersion
- outputs.files fileTree(dir: targetDir, include: "*.brk")
-
- doFirst {
- buildGlobals.execOps.javaexec {
- mainClass = "org.apache.lucene.analysis.icu.RBBIRuleCompiler"
- classpath = sourceSets.tools.runtimeClasspath
-
- ignoreExitValue = false
- enableAssertions true
- args = [sourceDir, targetDir]
- }
- }
- })
-
- tasks.register("compileIcuWindows", {
- doFirst {
- def v = icu4jVersion
- def icuBinZip = file("${icuBuildDir}/icu4c-${v.replace(".", "_")}.zip")
- if (!icuBinZip.exists()) {
- icuBuildDir.mkdirs()
-
- // Download binaries matching icu4j version in version.props
- def src = URI.create("https://github.com/unicode-org/icu/releases/download/release-${v.replace(".", "-")}/icu4c-${v.replace(".", "_")}-Win64-MSVC2019.zip")
-
- logger.lifecycle("Trying to download binary ICU version: ${v} from:\n ${src}")
- Files.write(icuBinZip.toPath(), src.toURL().openStream().bytes)
- logger.lifecycle("Downloaded ${icuBinZip.size()} bytes.")
- }
-
- // Unzip.
- buildGlobals.fileOps.copy({
- into icuBuildDir
- from zipTree(icuBinZip)
- })
- }
- })
-
- tasks.register("compileIcuLinux", {
- doFirst {
- if (Os.isFamily(Os.FAMILY_WINDOWS)) {
- throw new GradleException("ICU compilation not supported on Windows.")
- }
-
- def v = icu4jVersion
- def icuSrcTgz = file("${icuBuildDir}/icu4c-${v.replace(".", "_")}-src.tgz")
-
- // Download sources for version matching icu4j version in version.props
- if (!icuSrcTgz.exists()) {
- icuBuildDir.mkdirs()
- def src = URI.create("https://github.com/unicode-org/icu/releases/download/release-${v.replace(".", "-")}/icu4c-${v.replace(".", "_")}-src.tgz")
- logger.lifecycle("Trying to download and compile ICU version: ${v} from:\n ${src}")
- Files.write(icuSrcTgz.toPath(), src.toURL().openStream().bytes)
- logger.lifecycle("Downloaded ${icuSrcTgz.size()} bytes.")
- }
-
- def icuSrcDir = file("${icuBuildDir}/icu/source")
- buildGlobals.fileOps.delete(icuSrcDir)
-
- // Extract the tgz
- buildGlobals.quietExec(it, {
- executable "tar"
- workingDir icuBuildDir
- args = [
- "-zxf",
- icuSrcTgz
- ]
- })
-
- // Compile: (cd icu/source && ./configure --prefix=$(pwd) --enable-rpath && make -j4)
- buildGlobals.quietExec(it, {
- executable "sh"
-
- workingDir icuSrcDir
- environment("CFLAGS", "-O0")
- environment("CXXFLAGS", "-O0")
- args = [
- "configure",
- "--prefix=${icuSrcDir}",
- "--enable-rpath"
- ]
- })
-
- buildGlobals.quietExec(it, {
- executable "make"
- workingDir icuSrcDir
- args = [
- "-j${buildOptions['tests.jvms'].get()}"
- ]
- })
-
- // Test that the binaries work: derb -V
- logger.lifecycle("Compiled ICU, checking...")
- buildGlobals.quietExec(it, {
- executable "./derb"
- workingDir icuBinDir
- args = ["-V"]
- })
- }
- })
-}
-
-
-// Regenerates UnicodeProps.java
-configure(project(":lucene:analysis:common")) {
- setupIcuDependencies(project)
-
- tasks.register("regenerateUnicodeProps", {
- def icuConfig = configurations.icu_current
- def groovyConfig = configurations.groovy
-
- def outputFile = file("src/java/org/apache/lucene/analysis/util/UnicodeProps.java")
-
- description = "Regenerate ${outputFile} using ICU"
-
- dependsOn icuConfig
-
- inputs.property "icuConfig", icu4jVersion
- outputs.file outputFile
-
- doFirst {
- buildGlobals.execOps.javaexec({ JavaExecSpec spec ->
- spec.mainClass = "groovy.lang.GroovyShell"
- spec.classpath(icuConfig, groovyConfig)
-
- spec.args([
- "--encoding",
- "UTF-8",
- file("${resources}/GenerateUnicodeProps.groovy").toString(),
- outputFile.toString()
- ])
- })
- }
- })
-}
-
-// Regenerates CaseFolding.java
-configure(project(":lucene:core")) {
- setupIcuDependencies(project)
-
- tasks.register("generateUnicodeProps", {
- def icuConfig = configurations.icu_current
- def groovyConfig = configurations.groovy
- def outputFile = file("src/java/org/apache/lucene/util/automaton/CaseFolding.java")
-
- description = "Regenerate ${outputFile} using ICU"
-
- dependsOn icuConfig
-
- inputs.property "icuConfig", icu4jVersion
- outputs.file outputFile
-
- doFirst {
- buildGlobals.execOps.javaexec({
- mainClass = "groovy.lang.GroovyShell"
- classpath icuConfig, groovyConfig
-
- args = [
- "--encoding",
- "UTF-8",
- file("${resources}/GenerateCaseFolding.groovy"),
- outputFile
- ]
- })
- }
- })
-}
diff --git a/build-tools/build-infra/src/main/groovy/regenerate.javacc.gradle b/build-tools/build-infra/src/main/groovy/regenerate.javacc.gradle
deleted file mode 100644
index 73b6bf88f34..00000000000
--- a/build-tools/build-infra/src/main/groovy/regenerate.javacc.gradle
+++ /dev/null
@@ -1,329 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.nio.charset.Charset
-import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension
-
-
-// This adds javacc generation support.
-
-configure(rootProject) {
- tasks.register("javacc", {
- description = "Regenerate sources for corresponding javacc grammar files."
- group = "generation"
-
- dependsOn allprojects.collect { prj -> prj.tasks.withType(JavaCCTask) }
- })
-}
-
-def commonCleanups = { FileTree generatedFiles ->
- // This is a minor typo in a comment that nonetheless people have hand-corrected in the past.
- generatedFiles.matching({ include "CharStream.java" }).each {file ->
- buildGlobals.modifyFile(file, { text ->
- return text.replace(
- "implemetation",
- "implementation");
- })
- }
-
- generatedFiles.each {file ->
- buildGlobals.modifyFile(file, { text ->
- // Normalize EOLs and tabs (EOLs are a side-effect of modifyFile).
- text = text.replace("\t", " ");
- text = text.replaceAll("JavaCC - OriginalChecksum=[^*]+", "(filtered)")
- text = text.replace("StringBuffer", "StringBuilder")
- return text
- })
- }
-
- generatedFiles.matching({ include "*TokenManager.java" }).each { file ->
- buildGlobals.modifyFile(file, { text ->
- // Remove redundant imports.
- text = text.replaceAll(
- /(?m)^import .+/,
- "")
- // Add CharStream imports.
- text = text.replaceAll(
- /package (.+)/,
- '''
- package $1
- import org.apache.lucene.queryparser.charstream.CharStream;
- '''.trim())
- // Eliminates redundant cast message.
- text = text.replace(
- "int hiByte = (int)(curChar >> 8);",
- "int hiByte = curChar >> 8;")
- // Access to forbidden APIs.
- text = text.replace(
- "public java.io.PrintStream debugStream = System.out;",
- "// (debugStream omitted).")
- text = text.replace(
- "public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }",
- "// (setDebugStream omitted).")
- // Remove dangling doc comments
- text = text.replace(
- "/** Debug output. */",
- "// Debug output.")
- text = text.replace(
- "/** Set debug output. */",
- "// Set debug output.")
- return text
- })
- }
-}
-
-configure(project(":lucene:queryparser")) {
- configurations.create("javacc")
-
- dependencies {
- add("javacc", deps.javacc)
- }
-
- tasks.register("regenerateClassicQueryParser", JavaCCTask, {
- description = "Regenerate classic query parser from lucene/queryparser/classic/QueryParser.jj"
-
- javaccFile = file('src/java/org/apache/lucene/queryparser/classic/QueryParser.jj')
-
- afterGenerate << commonCleanups
- afterGenerate << { FileTree generatedFiles ->
- generatedFiles.matching { include "QueryParser.java" }.each { file ->
- buildGlobals.modifyFile(file, { text ->
- text = text.replace(
- "public QueryParser(CharStream ",
- "protected QueryParser(CharStream ")
- text = text.replace(
- "public QueryParser(QueryParserTokenManager ",
- "protected QueryParser(QueryParserTokenManager ")
- text = text.replace(
- "new java.util.ArrayList<int[]>",
- "new java.util.ArrayList<>")
- text = text.replace(
- "final private LookaheadSuccess jj_ls =",
- "static final private LookaheadSuccess jj_ls =")
- text = text.replace(
- "public class QueryParser ",
- '@SuppressWarnings({"unused","null"}) public class QueryParser ')
- text = text.replace(
- "final public Query TopLevelQuery(",
- "@Override final public Query TopLevelQuery(")
- text = text.replace(
- "public void ReInit(CharStream ",
- "@Override public void ReInit(CharStream ")
- return text
- })
- }
- }
- })
-
- tasks.register("regenerateSurroundQueryParser", JavaCCTask, {
- description = "Regenerate surround query parser from lucene/queryparser/surround/parser/QueryParser.jj"
-
- javaccFile = file('src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj')
-
- afterGenerate << commonCleanups
- afterGenerate << { FileTree generatedFiles ->
- generatedFiles.matching { include "QueryParser.java" }.each { file ->
- buildGlobals.modifyFile(file, { text ->
- text = text.replace(
- "import org.apache.lucene.analysis.TokenStream;",
- "")
- text = text.replace(
- "new java.util.ArrayList<int[]>",
- "new java.util.ArrayList<>")
- text = text.replace(
- "public class QueryParser ",
- '@SuppressWarnings({"unused","null"}) public class QueryParser ')
- return text
- })
- }
- }
- })
-
- tasks.register("regenerateFlexibleStandardSyntaxParser", JavaCCTask, {
- description = "Regenerate flexible query parser from queryparser/flexible/standard/parser/StandardSyntaxParser.jj"
-
- javaccFile = file('src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj')
-
- afterGenerate << commonCleanups
- afterGenerate << { FileTree generatedFiles ->
- generatedFiles.matching { include "ParseException.java" }.each { file ->
- buildGlobals.modifyFile(file, { text ->
- // Modify constructor.
- text = text.replace(
- "class ParseException extends Exception",
- "class ParseException extends QueryNodeParseException")
-
- // Modify imports.
- text = text.replace(
- "package org.apache.lucene.queryparser.flexible.standard.parser;", '''\
- package org.apache.lucene.queryparser.flexible.standard.parser;
-
- import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException;
- import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages;
- import org.apache.lucene.queryparser.flexible.messages.Message;
- import org.apache.lucene.queryparser.flexible.messages.MessageImpl;
- ''')
-
- // Modify constructors and code bits
- text = text.replaceAll(
- /(?s)[ ]*public ParseException\(Token currentTokenVal[^}]+[}]/, '''\
- public ParseException(Token currentTokenVal,
- int[][] expectedTokenSequencesVal, String[] tokenImageVal)
- {
- super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, initialise(
- currentTokenVal, expectedTokenSequencesVal, tokenImageVal)));
- this.currentToken = currentTokenVal;
- this.expectedTokenSequences = expectedTokenSequencesVal;
- this.tokenImage = tokenImageVal;
- }
- ''')
-
- text = text.replaceAll(
- /(?s)[ ]*public ParseException\(String message\)[^}]+[}]/, '''\
- public ParseException(Message message)
- {
- super(message);
- }
- ''')
-
- text = text.replaceAll(
- /(?s)[ ]*public ParseException\(\)[^}]+[}]/, '''\
- public ParseException()
- {
- super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, "Error"));
- }
- ''')
- return text
- })
- }
-
- generatedFiles.matching { include "StandardSyntaxParser.java" }.each { file ->
- buildGlobals.modifyFile(file, { text ->
- // Remove redundant cast
- text = text.replace(
- "new java.util.ArrayList<int[]>",
- "new java.util.ArrayList<>")
- text = text.replace(
- "new ArrayList<QueryNode>()",
- "new ArrayList<>()")
- text = text.replace(
- "Collections.<QueryNode> singletonList",
- "Collections.singletonList")
- text = text.replace(
- "public class StandardSyntaxParser ",
- '@SuppressWarnings({"unused","null"}) public class StandardSyntaxParser ')
- return text
- })
- }
- }
- })
-}
-
-// We always regenerate, no need to declare outputs.
-class JavaCCTask extends DefaultTask {
- @InputFile
- File javaccFile
-
- /**
- * Apply closures to all generated files before they're copied back
- * to mainline code.
- */
- // A subtle bug here is that this makes it not an input... should be a list of replacements instead?
- @Internal
- List<Closure<FileTree>> afterGenerate = new ArrayList<>()
-
- @OutputFiles
- List<File> getGeneratedSources() {
- // Return the list of generated files.
- def baseDir = javaccFile.parentFile
- def baseName = javaccFile.name.replace(".jj", "")
-
- return [
- project.file("${baseDir}/${baseName}.java"),
- project.file("${baseDir}/${baseName}Constants.java"),
- project.file("${baseDir}/${baseName}TokenManager.java"),
- project.file("${baseDir}/ParseException.java"),
- project.file("${baseDir}/Token.java"),
- project.file("${baseDir}/TokenMgrError.java")
- ]
- }
-
- JavaCCTask() {
- dependsOn(project.configurations.javacc)
- }
-
- @TaskAction
- def generate() {
- if (!javaccFile || !javaccFile.exists()) {
- throw new GradleException("Input file does not exist: ${javaccFile}")
- }
-
- def buildGlobals = project.getExtensions().getByType(LuceneBuildGlobalsExtension)
-
- // Run javacc generation into temporary folder so that we know all the generated files
- // and can post-process them easily.
- def tempDir = this.getTemporaryDir()
- tempDir.mkdirs()
- buildGlobals.fileOps.delete(project.fileTree(tempDir, { include: "**/*.java" }))
-
- def targetDir = javaccFile.parentFile
- logger.lifecycle("Recompiling JavaCC: ${project.rootDir.relativePath(javaccFile)}")
-
- def output = new ByteArrayOutputStream()
- def result = buildGlobals.execOps.javaexec({
- classpath {
- project.configurations.javacc
- }
-
- ignoreExitValue = true
- standardOutput = output
- errorOutput = output
-
- mainClass = "org.javacc.parser.Main"
- args += [
- "-OUTPUT_DIRECTORY=${tempDir}",
- javaccFile
- ]
- })
-
- // Unless we request verbose logging, don't emit javacc output.
- if (result.exitValue != 0) {
- throw new GradleException("JavaCC failed to compile ${javaccFile}, here is the compilation output:\n${output}")
- }
-
- // Make sure we don't have warnings.
- if (output.toString(Charset.defaultCharset()).contains("Warning:")) {
- throw new GradleException("JavaCC emitted warnings for ${javaccFile}, here is the compilation output:\n${output}")
- }
-
- // Apply any custom modifications.
- def generatedFiles = project.fileTree(tempDir)
-
- afterGenerate.each {closure ->
- closure.call(generatedFiles)
- }
-
- // Copy back to mainline sources.
- buildGlobals.fileOps.copy({ CopySpec spec ->
- spec.from tempDir
- spec.into targetDir
-
- // We don't need CharStream interface as we redirect to our own.
- spec.exclude "CharStream.java"
- })
- }
-}
diff --git a/build-tools/build-infra/src/main/groovy/regenerate.jflex.gradle b/build-tools/build-infra/src/main/groovy/regenerate.jflex.gradle
deleted file mode 100644
index 854f9f37c7e..00000000000
--- a/build-tools/build-infra/src/main/groovy/regenerate.jflex.gradle
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.gradle.plugins.misc.QuietExec
-import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension
-
-// Add a top-level pseudo-task to which we will attach individual regenerate tasks.
-
-def resources = rootProject.file("gradle/regenerate/jflex")
-def skeletonDefault = file("${resources}/skeleton.default.txt")
-def skeletonNoBufferExpansion = file("${resources}/skeleton.disable.buffer.expansion.txt")
-
-configure(project(":lucene:core")) {
- configurations {
- jflex
- }
-
- dependencies {
- jflex deps.jflex
- }
-
- tasks.register("regenerateStandardTokenizer", JFlexTask, {
- description = "Regenerate StandardTokenizerImpl.java"
-
- jflexFile = file('src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex')
- skeleton = skeletonNoBufferExpansion
-
- doLast {
- ant.replace(
- file: file('src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java'),
- encoding: "UTF-8",
- token: "private static final int ZZ_BUFFERSIZE =",
- value: "private int ZZ_BUFFERSIZE ="
- )
- }
- })
-}
-
-configure(project(":lucene:analysis:common")) {
- configurations {
- jflex
- }
-
- dependencies {
- jflex deps.jflex
- }
-
- def regenerateTlds = tasks.register("regenerateTlds", {
- def tldZones = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
- def jflexMacro = file("src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex")
- def tldList = file("src/test/org/apache/lucene/analysis/email/TLDs.txt")
-
- description = "Regenerate top-level domain jflex macros and tests"
-
- dependsOn { sourceSets.tools.runtimeClasspath }
-
- inputs.property "tldZones", tldZones
- outputs.files jflexMacro, tldList
-
- doFirst {
- File tmpJflexMacro = File.createTempFile(jflexMacro.getName(), ".tmp", getTemporaryDir())
- File tmpTldList = File.createTempFile(tldList.getName(), ".tmp", getTemporaryDir())
-
- project.extensions.buildGlobals.execOps.javaexec {
- mainClass = "org.apache.lucene.analysis.standard.GenerateJflexTLDMacros"
- classpath = sourceSets.tools.runtimeClasspath
-
- ignoreExitValue = false
- args = [
- tldZones,
- tmpJflexMacro,
- tmpTldList
- ]
- }
-
- // LUCENE-9926: tldZones is regenerated daily. Compare the generated content (excluding comments) so that
- // we only update actual output files if non-comments have changed.
- def contentLines = { File file ->
- if (file.exists()) {
- List<String> lines = file.readLines("UTF-8")
- lines.removeIf { line -> line.isBlank() || line.startsWith("//") }
- return lines
- } else {
- return []
- }
- }
-
- if (contentLines(tmpTldList).equals(contentLines(tldList))) {
- logger.lifecycle("Generated TLD content identical as before, not updating.")
- } else {
- tldList.setBytes tmpTldList.bytes
- jflexMacro.setBytes tmpJflexMacro.bytes
- logger.lifecycle("You've regenerated the TLD include file, remember to regenerate UAX29URLEmailTokenizerImpl too.")
- }
- }
- })
-
- tasks.register("regenerateWikipediaTokenizer", JFlexTask, {
- description = "Regenerate WikipediaTokenizerImpl.java"
-
- jflexFile = file('src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex')
- skeleton = skeletonDefault
- })
-
- tasks.register("regenerateClassicTokenizer", JFlexTask, {
- description = "Regenerate ClassicTokenizerImpl.java"
-
- jflexFile = file('src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.jflex')
- skeleton = skeletonDefault
- })
-
- tasks.register("regenerateUAX29URLEmailTokenizer", JFlexTask, {
- description = "Regenerate UAX29URLEmailTokenizerImpl.java"
-
- // UAX29URLEmailTokenizerImpl.jflex includes: ASCIITLD.jflex
- // so we make sure it is up to date. Also, make sure the ordering of internal
- // tasks is enforced (just scheduling top-level wrappers doesn't mean their subtask graphs cannot be reordered).
- dependsOn regenerateTlds
-
- jflexFile = file('src/java/org/apache/lucene/analysis/email/UAX29URLEmailTokenizerImpl.jflex')
- skeleton = skeletonNoBufferExpansion
- heapSize = "12g"
-
- // Add included files as inputs.
- inputs.file file('src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex')
-
- doFirst {
- logger.lifecycle("Regenerating UAX29URLEmailTokenizerImpl. This may take a long time (and requires ${heapSize} of memory!).")
- }
-
- doLast {
- ant.replace(
- file: file('src/java/org/apache/lucene/analysis/email/UAX29URLEmailTokenizerImpl.java'),
- encoding: "UTF-8",
- token: "private static final int ZZ_BUFFERSIZE =",
- value: "private int ZZ_BUFFERSIZE ="
- )
- }
- })
-
- def generateHTMLCharacterEntities = tasks.register("regenerateHTMLCharacterEntities", QuietExec, {
- description = "Regenerate HTMLCharacterEntities.jflex"
-
- def target = file('src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex')
- def script = file("${resources}/htmlentity.py")
-
- outputs.files target
- inputs.file script
-
- executable = buildGlobals.externalTool("python3")
- workingDir = target.parentFile
- args += [
- "-B",
- // don't write any bytecode cache
- script,
- target
- ]
-
- doLast {
- project.ant.fixcrlf(
- file: target,
- encoding: "UTF-8",
- eol: "lf"
- )
- }
- })
-
- tasks.register("regenerateHTMLStripCharFilter", JFlexTask, {
- description = "Regenerate HTMLStripCharFilter.java"
-
- // HTMLStripCharFilter.jflex includes HTMLCharacterEntities.jflex so we make sure it's up to date.
- dependsOn generateHTMLCharacterEntities
-
- // Add included files as inputs.
- inputs.file file('src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex')
-
- jflexFile = file('src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex')
- skeleton = skeletonDefault
- })
-}
-
-class JFlexTask extends DefaultTask {
- @InputFile
- File jflexFile
-
- @InputFile
- File skeleton
-
- @Internal
- String heapSize
-
- @OutputFile
- File getGeneratedFile() {
- return project.file(jflexFile.absolutePath.replace(".jflex", ".java"))
- }
-
- JFlexTask() {
- dependsOn(project.configurations.jflex)
- }
-
- @TaskAction
- def generate() {
- if (!jflexFile || !jflexFile.exists()) {
- throw new GradleException("JFlex file does not exist: ${jflexFile}")
- }
-
- def target = project.file(jflexFile.absolutePath.replace(".jflex", ".java"))
-
- logger.lifecycle("Recompiling JFlex: ${project.rootDir.relativePath(jflexFile)}")
-
- project.extensions.buildGlobals.execOps.javaexec {
- classpath {
- project.configurations.jflex
- }
-
- mainClass = "jflex.Main"
- args += [
- "-nobak",
- "--quiet",
- "--encoding",
- "UTF-8",
- ]
-
- if (heapSize) {
- maxHeapSize = heapSize
- }
-
- if (skeleton) {
- args += [
- "--skel",
- skeleton.absolutePath
- ]
- }
-
- args += [
- "-d",
- target.parentFile.absolutePath,
- jflexFile
- ]
- }
-
- // fix invalid SuppressWarnings from jflex, so that it works with javac.
- // we also need to suppress 'unused' because of dead code, so it passes ecjLint
- // you can't "stack" SuppressWarnings annotations, jflex adds its own, this is the only way
- // https://github.com/jflex-de/jflex/issues/762
- project.ant.replace(
- file: target,
- encoding: "UTF-8",
- token: 'SuppressWarnings("fallthrough")',
- value: 'SuppressWarnings({"fallthrough","unused"})'
- )
-
- // Seems like a bug in gjf. https://github.com/google/google-java-format/issues/1260
- project.ant.replace(
- file: target,
- encoding: "UTF-8",
- token: '/* Break so we don\'t hit fall-through warning: */',
- value: ''
- )
- }
-}
diff --git a/build-tools/build-infra/src/main/groovy/regenerate.kuromoji.gradle b/build-tools/build-infra/src/main/groovy/regenerate.kuromoji.gradle
deleted file mode 100644
index 7c55c357ef7..00000000000
--- a/build-tools/build-infra/src/main/groovy/regenerate.kuromoji.gradle
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import de.undercouch.gradle.tasks.download.Download
-
-// This downloads and compiles Kuromoji dictionaries.
-
-def recompileDictionary(project, dictionaryName, Closure closure) {
- buildGlobals.execOps.javaexec {
- mainClass = "org.apache.lucene.analysis.ja.dict.DictionaryBuilder"
- classpath = project.sourceSets.main.runtimeClasspath
-
- jvmArgs '-Xmx1G'
-
- with closure
- }
- project.logger.lifecycle("Automaton regenerated from dictionary: ${dictionaryName}")
-}
-
-configure(project(":lucene:analysis:kuromoji")) {
- plugins.withType(JavaPlugin).configureEach {
- project.ext {
- targetDir = project.file("src/resources")
- }
-
- def mecabDictionaryName = "mecab-ipadic-2.7.0-20070801"
- // original source: "https://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/${mecabDictionaryName}.tar.gz"
- def mecabDictionarySource = "https://s3.amazonaws.com/lucene-testdata/mecab/${mecabDictionaryName}.tar.gz"
-
- def compileMecab = tasks.register("compileMecab", Download, {
- dependsOn sourceSets.main.runtimeClasspath
-
- def dictionaryFile = file("${buildDir}/generate/${mecabDictionaryName}.tar.gz")
- def unpackedDir = file("${buildDir}/generate/${mecabDictionaryName}")
-
- src mecabDictionarySource
- dest dictionaryFile
- onlyIfModified true
-
- doLast {
- buildGlobals.fileOps.delete(sourceSets.main.resources.srcDirs.collect {File location ->
- fileTree(dir: location, include: "org/apache/lucene/analysis/ja/dict/*.dat")
- })
-
- // Unpack the downloaded archive.
- delete unpackedDir
- ant.untar(src: dictionaryFile, dest: unpackedDir, compression: "gzip") {
- ant.cutdirsmapper(dirs: "1")
- }
-
- // Apply patch via local git.
- buildGlobals.quietExec(it, {
- workingDir = unpackedDir
- executable "git"
- args += [
- "apply",
- project.file("src/tools/patches/Noun.proper.csv.patch").absolutePath
- ]
- })
-
- // Compile the dictionary
- recompileDictionary(project, mecabDictionaryName, {
- args += [
- "ipadic",
- unpackedDir,
- targetDir,
- "euc-jp",
- false
- ]
- })
- }
- })
-
- tasks.register("regenerateDictionariesFromMecab", {
- description = "Regenerate dictionaries from Mecab data"
-
- dependsOn compileMecab
-
- inputs.property("dictionarySource", mecabDictionarySource)
- outputs.files(fileTree(dir: "src/resources/org/apache/lucene/analysis/ja/dict", include: "*.dat"))
-
- regenerate {
- // No need to run any follow-ups, we generate binary files (not sources).
- followedUpBy = []
- ifSkippedAlsoSkip = ["compileMecab"]
- }
- })
-
- // TODO: this regenerates data files over the same files as mecab, above. This seems odd and inconsistent?
-
- // original source (down): https://rwthaachen.dl.osdn.jp/naist-jdic/53500/mecab-naist-jdic-0.6.3b-20111013.tar.gz
- def naistDictionaryName = "mecab-naist-jdic-0.6.3b-20111013"
- def naistDictionarySource = "https://s3.amazonaws.com/lucene-testdata/mecab/${naistDictionaryName}.tar.gz"
- tasks.register("compileNaist", Download, {
- description = "Recompile dictionaries from Naist data."
-
- dependsOn sourceSets.main.runtimeClasspath
-
- def dictionaryFile = file("${buildDir}/generate/${naistDictionaryName}.tar.gz")
- def unpackedDir = file("${buildDir}/generate/${naistDictionaryName}")
-
- src naistDictionarySource
- dest dictionaryFile
- onlyIfModified true
-
- doLast {
- buildGlobals.fileOps.delete(sourceSets.main.resources.srcDirs.collect {File location ->
- fileTree(dir: location, include: "org/apache/lucene/analysis/ja/dict/*.dat")
- })
-
- // Unpack the downloaded archive.
- delete unpackedDir
- ant.untar(src: dictionaryFile, dest: unpackedDir, compression: "gzip") {
- ant.cutdirsmapper(dirs: "1")
- }
-
- // Compile the dictionary
- recompileDictionary(project, naistDictionaryName, {
- args += [
- "ipadic",
- unpackedDir,
- targetDir,
- "euc-jp",
- false
- ]
- })
- }
- })
- }
-}
diff --git a/build-tools/build-infra/src/main/groovy/regenerate.moman.gradle b/build-tools/build-infra/src/main/groovy/regenerate.moman.gradle
deleted file mode 100644
index 936e34c0c8f..00000000000
--- a/build-tools/build-infra/src/main/groovy/regenerate.moman.gradle
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import de.undercouch.gradle.tasks.download.Download
-
-def resources = rootProject.file("gradle/regenerate/moman")
-
-configure(project(":lucene:core")) {
-
- tasks.register("regeneratePacked64SingleBlock", {
- description("Regenerate Packed64SingleBlock and BulkOperation*.java")
-
- def targetDir = file("src/java/org/apache/lucene/util/packed")
-
- def templateFiles = [
- file("${resources}/gen_BulkOperation.py"),
- file("${resources}/gen_Packed64SingleBlock.py")
- ]
-
- inputs.files templateFiles
- outputs.files fileTree(dir: targetDir, includes: [
- "Packed64SingleBlock.java",
- "BulkOperation*.java"
- ])
-
- doLast {
- templateFiles.each {prog ->
- logger.lifecycle("Executing: ${prog} in ${targetDir}")
- buildGlobals.quietExec(it, {
- workingDir = targetDir
- executable = buildGlobals.externalTool("python3")
- args = ['-B', "${prog}"]
- })
- }
- // Correct line endings for Windows.
- project.ant.fixcrlf(
- srcDir: targetDir,
- includes: 'Packed64SingleBlock.java, BulkOperation*.java',
- encoding: 'UTF-8',
- eol: 'lf'
- )
- }
- })
-
- def momanSource = "https://github.com/jpbarrette/moman/archive/497c90e34e412b6494db6dabf0d95db8034bd325.zip"
- def momanDir = file("${buildDir}/moman")
-
- def installMoman = tasks.register("installMoman", Download, {
- def momanZip = file("${momanDir}/moman.zip")
-
- src momanSource
- dest momanZip
- onlyIfModified true
-
- doLast {
- ant.unzip(src: momanZip, dest: momanDir, overwrite: "true") {
- ant.cutdirsmapper(dirs: "1")
- }
- }
- })
-
- tasks.register("regenerateParametricDescription", {
- dependsOn installMoman
-
- description("Regenerate *ParametricDescription")
- regenerate {
- ifSkippedAlsoSkip = ["installMoman"]
- }
-
- def targetDir = file("src/java/org/apache/lucene/util/automaton")
- def templateFile = file("${resources}/createLevAutomata.py")
-
- inputs.property("source", momanSource)
- inputs.file templateFile
- outputs.files fileTree(dir: targetDir, includes: ["*ParametricDescription.java"])
-
- doLast {
- ['1', '2'].each {num ->
- ['True', 'False'].each {transpose ->
- buildGlobals.quietExec(it, {
- workingDir = targetDir
- executable = buildGlobals.externalTool("python3")
- args = [
- '-B',
- templateFile.toString(),
- num,
- transpose,
- "${momanDir}/finenight/python"
- ]
- })
- }
- }
- project.ant.fixcrlf(
- srcDir: targetDir,
- includes: '*ParametricDescription.java',
- encoding: 'UTF-8',
- eol: 'lf'
- )
- }
- })
-}
diff --git a/build-tools/build-infra/src/main/groovy/regenerate.nori.gradle b/build-tools/build-infra/src/main/groovy/regenerate.nori.gradle
deleted file mode 100644
index 155ce3884ad..00000000000
--- a/build-tools/build-infra/src/main/groovy/regenerate.nori.gradle
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import de.undercouch.gradle.tasks.download.Download
-
-// This downloads and compiles Nori dictionaries.
-
-def recompileDictionary(project, dictionaryName, Closure closure) {
- buildGlobals.execOps.javaexec {
- mainClass = "org.apache.lucene.analysis.ko.dict.DictionaryBuilder"
- classpath = project.sourceSets.main.runtimeClasspath
-
- jvmArgs '-Xmx1G'
-
- with closure
- }
- project.logger.lifecycle("Automaton regenerated from dictionary: ${dictionaryName}")
-}
-
-configure(project(":lucene:analysis:nori")) {
- plugins.withType(JavaPlugin).configureEach {
- project.ext {
- targetDir = project.file("src/resources")
- }
-
- def dictionaryName = "mecab-ko-dic-2.1.1-20180720"
- def dictionarySource = "https://s3.amazonaws.com/lucene-testdata/mecab/${dictionaryName}.tar.gz"
- def compileMecabKo = tasks.register("compileMecabKo", Download, {
-
- dependsOn sourceSets.main.runtimeClasspath
-
- def dictionaryFile = file("${buildDir}/generate/${dictionaryName}.tar.gz")
- def unpackedDir = file("${buildDir}/generate/${dictionaryName}")
-
- src dictionarySource
- dest dictionaryFile
- onlyIfModified true
-
- doLast {
- buildGlobals.fileOps.delete(sourceSets.main.resources.srcDirs.collect {File location ->
- fileTree(dir: location, include: "org/apache/lucene/analysis/ja/dict/*.dat")
- })
-
- // Unpack the downloaded archive.
- delete unpackedDir
- ant.untar(src: dictionaryFile, dest: unpackedDir, compression: "gzip") {
- ant.cutdirsmapper(dirs: "1")
- }
-
- // Compile the dictionary
- recompileDictionary(project, dictionaryName, {
- args += [
- unpackedDir,
- targetDir,
- "utf-8",
- false
- ]
- })
- }
- })
-
- tasks.register("regenerateDictionariesFromMecabKo", {
- description = "Recompile dictionaries from Mecab-Ko data."
-
- dependsOn compileMecabKo
-
- inputs.property("dictionarySource", dictionarySource)
- outputs.files(fileTree(dir: "src/resources/org/apache/lucene/analysis/ko/dict", include: "*.dat"))
-
- regenerate {
- // No need to run any follow-ups, we generate binary files (not sources).
- followedUpBy = []
- ifSkippedAlsoSkip = ["compileMecabKo"]
- }
- })
- }
-}
diff --git a/build-tools/build-infra/src/main/groovy/regenerate.snowball.gradle b/build-tools/build-infra/src/main/groovy/regenerate.snowball.gradle
deleted file mode 100644
index 185a7f2bfa1..00000000000
--- a/build-tools/build-infra/src/main/groovy/regenerate.snowball.gradle
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import org.apache.lucene.gradle.plugins.misc.QuietExec
-import de.undercouch.gradle.tasks.download.Download
-import org.apache.tools.ant.taskdefs.condition.Os
-
-def resources = rootProject.file("gradle/regenerate/snowball")
-
-configure(project(":lucene:analysis:common")) {
- // git commit hash of source code https://github.com/snowballstem/snowball/
- def snowballStemmerCommit = "34f3612e5e8c48975243bc2e87561abdac5aa9bb"
- // git commit hash of stopwords https://github.com/snowballstem/snowball-website
- def snowballWebsiteCommit = "424fd1f75044160bb4a92f1daf08cce618459374"
-
- def snowballWorkDir = file("${buildDir}/snowball")
-
- def snowballStemmerDir = file("${snowballWorkDir}/stemmers-${snowballStemmerCommit}")
- def snowballWebsiteDir = file("${snowballWorkDir}/website-${snowballWebsiteCommit}")
-
- def snowballScript = file("${resources}/snowball.sh")
-
- def unpackFromZip = { zipFile, targetDir ->
- buildGlobals.fileOps.sync({SyncSpec spec ->
- spec.from(zipTree(zipFile), {
- eachFile { fcd ->
- fcd.relativePath = new RelativePath(true, fcd.relativePath.segments.drop(1))
- }
- })
- spec.into targetDir
- })
- }
-
- // downloads snowball stemmers (or use a cached copy)
- def downloadSnowballStemmers = tasks.register("downloadSnowballStemmers", Download, {
- src "https://github.com/snowballstem/snowball/archive/${snowballStemmerCommit}.zip"
- dest file("${snowballStemmerDir}.zip")
- overwrite false
- tempAndMove true
-
- doLast {
- unpackFromZip(dest, snowballStemmerDir)
- }
- })
-
- // downloads snowball website (or use a cached copy)
- def downloadSnowballWebsite = tasks.register("downloadSnowballWebsite", Download, {
- src "https://github.com/snowballstem/snowball-website/archive/${snowballWebsiteCommit}.zip"
- def snowballWebsiteZip = file("${snowballWebsiteDir}.zip")
- dest snowballWebsiteZip
- overwrite false
- tempAndMove true
-
- doLast {
- unpackFromZip(snowballWebsiteZip, snowballWebsiteDir)
- }
- })
-
- // runs a shell script to regenerate stemmers, base stemming subclasses, test data, and stopwords.
- tasks.register("regenerateSnowball", QuietExec, {
- description = "Regenerate snowball stemmers."
-
- dependsOn downloadSnowballStemmers
- dependsOn downloadSnowballWebsite
-
- regenerate {
- ifSkippedAlsoSkip = [
- downloadSnowballStemmers.name,
- downloadSnowballWebsite.name
- ]
- }
-
- inputs.files fileTree(
- dir: "src/java/org/tartarus/snowball",
- include: [
- "Among.java",
- "SnowballStemmer.java",
- "SnowballProgram.java",
- "ext/*Stemmer.java"
- ])
-
- inputs.files fileTree(
- dir: "src/resources/org/apache/lucene/analysis/snowball",
- include: "*_stop.txt")
-
- // Don't bother on Windows, won't work.
- if (Os.isFamily(Os.FAMILY_WINDOWS)) {
- doFirst {
- // Just emit a big fat error message. Fail the build so that checksums are not regenerated.
- throw new GradleException("Snowball generation does not work on Windows (bash must be available).")
- }
- }
-
- executable "bash"
- args = [
- snowballScript,
- snowballStemmerDir,
- snowballWebsiteDir,
- projectDir
- ]
- })
-}
diff --git a/build-tools/build-infra/src/main/groovy/regenerate.unicode-test-classes.gradle b/build-tools/build-infra/src/main/groovy/regenerate.unicode-test-classes.gradle
deleted file mode 100644
index 43d8d01b738..00000000000
--- a/build-tools/build-infra/src/main/groovy/regenerate.unicode-test-classes.gradle
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.gradle.plugins.misc.QuietExec
-
-// Regenerates test classes from unicode data.
-
-configure(project(":lucene:test-framework")) {
- tasks.register("regenerateEmojiTokenizationTest", QuietExec, {
- def unicodeVersion = "12.1"
-
- def genDir = file("src/java/org/apache/lucene/tests/analysis/standard")
- def genScript = file("${genDir}/generateEmojiTokenizationTest.pl")
- def genOutput = file("${genDir}/EmojiTokenizationTestUnicode_${unicodeVersion.replace('.', '_')}.java")
-
- description = "Regenerate ${genOutput}"
-
- inputs.file genScript
- inputs.property "unicodeVersion", unicodeVersion
- outputs.file genOutput
-
- workingDir = genDir
- executable = buildGlobals.externalTool("perl")
- args = [
- genScript,
- "-v",
- unicodeVersion
- ]
- })
-
- tasks.register("regenerateJavaUnicodeWordBreakTest", QuietExec, {
- def unicodeVersion = "12.1"
-
- def genDir = file("src/java/org/apache/lucene/tests/analysis/standard")
- def genScript = file("${genDir}/generateJavaUnicodeWordBreakTest.pl")
- def genOutput = file("${genDir}/WordBreakTestUnicode_${unicodeVersion.replace('.', '_')}.java")
-
- description = "Regenerate ${genOutput}"
-
- inputs.file genScript
- inputs.property "unicodeVersion", unicodeVersion
- outputs.file genOutput
-
- workingDir = genDir
- executable = buildGlobals.externalTool("perl")
- args = [
- genScript,
- "-v",
- unicodeVersion
- ]
- })
-}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/WipeGradleTempPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/WipeGradleTempPlugin.java
index 266d1a18549..2b02a84ce5c 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/WipeGradleTempPlugin.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/WipeGradleTempPlugin.java
@@ -16,6 +16,7 @@
*/
package org.apache.lucene.gradle.plugins.hacks;
+import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
@@ -25,13 +26,16 @@ import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Stream;
-import javax.inject.Inject;
import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;
import org.gradle.api.Project;
-import org.gradle.api.file.FileTree;
-import org.gradle.api.internal.file.FileOperations;
+import org.gradle.api.file.Directory;
+import org.gradle.api.file.DirectoryProperty;
import org.gradle.api.invocation.Gradle;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.services.BuildService;
+import org.gradle.api.services.BuildServiceParameters;
import org.gradle.api.tasks.Delete;
import org.gradle.api.tasks.TaskCollection;
import org.gradle.api.tasks.TaskContainer;
@@ -43,14 +47,33 @@ import org.gradle.api.tasks.testing.Test;
* @see "LUCENE-9471"
*/
public abstract class WipeGradleTempPlugin extends LuceneGradlePlugin {
- @Inject
- public abstract FileOperations getFileOps();
-
@Override
public void apply(Project rootProject) {
applicableToRootProjectOnly(rootProject);
- cleanTempFilesAfterBuildFinished(rootProject);
+ Gradle gradle = rootProject.getGradle();
+ gradle
+ .getSharedServices()
+ .registerIfAbsent(
+ "gradleTempCleanupService",
+ TempCleanupService.class,
+ service -> {
+ service.parameters(
+ params -> {
+ params.getProjectRootDir().set(getProjectRootPath(rootProject).toFile());
+ File gradleUserHome = gradle.getGradleUserHomeDir();
+ params.getUserHomeDir().set(gradleUserHome);
+ params
+ .getGradleDaemonDir()
+ .set(
+ gradleUserHome
+ .toPath()
+ .resolve("daemon")
+ .resolve(gradle.getGradleVersion())
+ .toFile());
+ });
+ })
+ .get(); // make sure it'll run.
rootProject
.getAllprojects()
@@ -93,72 +116,84 @@ public abstract class WipeGradleTempPlugin extends LuceneGradlePlugin {
});
}
- private void cleanTempFilesAfterBuildFinished(Project rootProject) {
- rootProject
- .getGradle()
- .buildFinished(
- _ -> {
- cleanUpRedirectedTmpDir(rootProject);
- cleanUpGradlesTempDir(rootProject);
- });
- }
+ public abstract static class TempCleanupService
+ implements BuildService<TempCleanupServiceParams>, AutoCloseable {
+ private static final Logger LOGGER = Logging.getLogger(TempCleanupService.class);
- private static void cleanUpGradlesTempDir(Project rootProject) {
- try {
- // clean up any files older than 3 hours from the user's gradle temp. the time
- // limit is added so that we don't interfere with any concurrent builds... just in
- // case.
- Instant deadline = Instant.now().minus(3, ChronoUnit.HOURS);
- List<Stream<Path>> toDelete = new ArrayList<>();
+ @Override
+ public void close() {
+ cleanUpRedirectedTmpDir(getParameters().getProjectRootDir().get());
+ cleanUpGradlesTempDir(
+ getParameters().getUserHomeDir().get(), getParameters().getGradleDaemonDir().get());
+ }
- Gradle gradle = rootProject.getGradle();
- Path gradleUserHome = gradle.getGradleUserHomeDir().toPath();
+ private void cleanUpGradlesTempDir(Directory gradleUserHomeDir, Directory gradleDaemonDir) {
+ try {
+ // clean up any files older than 3 hours from the user's gradle temp. the time
+ // limit is added so that we don't interfere with any concurrent builds... just in
+ // case.
+ Instant deadline = Instant.now().minus(3, ChronoUnit.HOURS);
+ List<Stream<Path>> toDelete = new ArrayList<>();
- var gradleTmp = gradleUserHome.resolve(".tmp");
- if (Files.exists(gradleTmp)) {
- toDelete.add(Files.list(gradleTmp));
- }
+ Path gradleUserHome = gradleUserHomeDir.getAsFile().toPath();
- var daemonDir = gradleUserHome.resolve("daemon").resolve(gradle.getGradleVersion());
- if (Files.exists(daemonDir)) {
- toDelete.add(Files.list(daemonDir).filter(path -> path.toString().endsWith(".out.log")));
- }
+ var gradleTmp = gradleUserHome.resolve(".tmp");
+ LOGGER.info("Cleaning up gradle's user temp dir: {}", gradleTmp);
+ if (Files.exists(gradleTmp)) {
+ toDelete.add(Files.list(gradleTmp));
+ }
- for (Stream<Path> stream : toDelete) {
- try {
- for (var path : stream.toList()) {
- if (!Files.isRegularFile(path)) {
- continue;
- }
- if (!Files.getLastModifiedTime(path).toInstant().isBefore(deadline)) {
- continue;
+ var daemonDir = gradleDaemonDir.getAsFile().toPath();
+ LOGGER.info("Cleaning up gradle's daemon logs dir: {}", daemonDir);
+ if (Files.exists(daemonDir)) {
+ toDelete.add(Files.list(daemonDir).filter(path -> path.toString().endsWith(".out.log")));
+ }
+
+ for (Stream<Path> stream : toDelete) {
+ try {
+ for (var path : stream.toList()) {
+ if (!Files.isRegularFile(path)) {
+ continue;
+ }
+ if (!Files.getLastModifiedTime(path).toInstant().isBefore(deadline)) {
+ continue;
+ }
+ Files.deleteIfExists(path);
}
- Files.deleteIfExists(path);
+ } finally {
+ stream.close();
}
- } finally {
- stream.close();
}
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ private void cleanUpRedirectedTmpDir(Directory projectRootDir) {
+ // Clean up the java.io.tmpdir we've redirected gradle to use (LUCENE-9471).
+ // these are still used and populated with junk.
+ try {
+ try (var files = Files.list(projectRootDir.dir(".gradle/tmp").getAsFile().toPath())) {
+ var workerPaths =
+ files
+ .filter(
+ path -> path.getFileName().toString().startsWith("gradle-worker-classpath"))
+ .toList();
+ for (var path : workerPaths) {
+ Files.delete(path);
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
- } catch (IOException e) {
- throw new UncheckedIOException(e);
}
}
- private void cleanUpRedirectedTmpDir(Project rootProject) {
- // Clean up the java.io.tmpdir we've redirected gradle to use (LUCENE-9471).
- // these are still used and populated with junk.
- FileTree tempFiles =
- rootProject
- .fileTree(".gradle/tmp")
- .matching(
- patternFilterable -> {
- patternFilterable.include("gradle-worker-classpath*");
- });
- getFileOps()
- .delete(
- spec -> {
- spec.delete(tempFiles);
- spec.setFollowSymlinks(false);
- });
+ public interface TempCleanupServiceParams extends BuildServiceParameters {
+ DirectoryProperty getProjectRootDir();
+
+ DirectoryProperty getUserHomeDir();
+
+ DirectoryProperty getGradleDaemonDir();
}
}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ErrorPronePlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ErrorPronePlugin.java
index 58fc2f03d12..af5589bd5c0 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ErrorPronePlugin.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ErrorPronePlugin.java
@@ -103,9 +103,9 @@ public class ErrorPronePlugin extends LuceneGradlePlugin {
skipReason = "skipped explicitly on a CI build it seems";
} else {
skipReason =
- "skipped on builds not running inside CI environments, pass -P"
- + OPT_VALIDATION_ERRORPRONE
- + "=true to enable";
+ "skipped on non-CI environments, pass "
+ + ("-P" + OPT_VALIDATION_ERRORPRONE + "=true")
+ + " to enable";
}
}
}
@@ -116,6 +116,15 @@ public class ErrorPronePlugin extends LuceneGradlePlugin {
.register(
TASK_ERROR_PRONE_SKIPPED,
task -> {
+ boolean hasCheckTask =
+ project.getGradle().getStartParameter().getTaskNames().contains("check");
+
+ task.onlyIf(
+ _ -> {
+ // Only complain if we're running a 'check'.
+ return hasCheckTask;
+ });
+
task.doFirst(
t -> {
t.getLogger().warn("Errorprone linting turned off ({})", skipReason);
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckJarChecksumsAndLicensesPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckJarChecksumsAndLicensesPlugin.java
index 2c1c6ee6255..42c650836a3 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckJarChecksumsAndLicensesPlugin.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckJarChecksumsAndLicensesPlugin.java
@@ -36,7 +36,6 @@ import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;
import org.gradle.api.GradleException;
import org.gradle.api.Project;
import org.gradle.api.file.Directory;
-import org.gradle.api.file.FileTree;
import org.gradle.api.plugins.JavaPlugin;
import org.gradle.api.tasks.TaskContainer;
@@ -141,20 +140,14 @@ public class CheckJarChecksumsAndLicensesPlugin extends LuceneGradlePlugin {
}
// Dangling files (not referenced)
- Set<File> allExisting =
- task.getProject()
- .fileTree(
- licensesDir,
- (cfg) -> {
- cfg.exclude(
- Arrays.asList(
- "elegant-icon-font-*", // Used by Luke.
- "glove-LICENSE-PDDL.txt" // glove knn dictionary.
- ));
- })
- .getFiles();
+ var allExisting = task.getFileOperations().fileTree(licensesDir);
+ allExisting.exclude(
+ Arrays.asList(
+ "elegant-icon-font-*", // Used by Luke.
+ "glove-LICENSE-PDDL.txt" // glove knn dictionary.
+ ));
- List<File> dangling = new ArrayList<>(allExisting);
+ List<File> dangling = new ArrayList<>(allExisting.getFiles());
dangling.removeAll(referencedFiles);
dangling.sort(Comparator.comparing(File::getPath));
@@ -214,9 +207,9 @@ public class CheckJarChecksumsAndLicensesPlugin extends LuceneGradlePlugin {
candidates.add(new File(licensesDir, baseName + "-LICENSE-[type].txt"));
var finalBaseName = baseName;
- FileTree tree =
- task.getProject()
- .fileTree(licensesDir, cfg -> cfg.include(finalBaseName + "-LICENSE-*.txt"));
+ var tree = task.getFileOperations().fileTree(licensesDir);
+ tree.include(finalBaseName + "-LICENSE-*.txt");
+
found.addAll(tree.getFiles());
String prefix = baseName.replaceAll("[\\-][^-]+$", "");
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CollectJarInfos.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CollectJarInfos.java
index 60674013a1e..461545b0185 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CollectJarInfos.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CollectJarInfos.java
@@ -18,8 +18,10 @@ package org.apache.lucene.gradle.plugins.licenses;
import java.util.List;
import java.util.stream.Collectors;
+import javax.inject.Inject;
import org.gradle.api.DefaultTask;
import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.internal.file.FileOperations;
import org.gradle.api.tasks.InputFiles;
import org.gradle.api.tasks.Internal;
@@ -31,6 +33,9 @@ public abstract class CollectJarInfos extends DefaultTask {
@InputFiles
public abstract ConfigurableFileCollection getJarInfos();
+ @Inject
+ protected abstract FileOperations getFileOperations();
+
@Internal
protected List<JarInfo> getUniqueJarInfos() {
return getJarInfos().getFiles().stream()
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/LuceneRootConfigurationPlugin.java
similarity index 98%
rename from build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java
rename to build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/LuceneRootConfigurationPlugin.java
index 508948f2d07..62b9dc99f59 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/LuceneRootConfigurationPlugin.java
@@ -48,11 +48,14 @@ import org.gradle.api.tasks.TaskContainer;
import org.gradle.api.tasks.bundling.AbstractArchiveTask;
/** Initialization of the root project, globals, etc. */
-public class RootProjectSetupPlugin extends LuceneGradlePlugin {
+public class LuceneRootConfigurationPlugin extends LuceneGradlePlugin {
@Override
public void apply(Project rootProject) {
applicableToRootProjectOnly(rootProject);
+ PluginContainer plugins = rootProject.getPlugins();
+ plugins.apply(CheckEnvironmentPlugin.class);
+
// Register these plugins early, for everything.
rootProject
.getAllprojects()
@@ -64,7 +67,6 @@ public class RootProjectSetupPlugin extends LuceneGradlePlugin {
});
// Register other root-level plugins.
- PluginContainer plugins = rootProject.getPlugins();
plugins.apply(RegisterBuildGlobalsPlugin.class);
plugins.apply(GitInfoPlugin.class);
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/JFlexTask.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/JFlexTask.java
new file mode 100644
index 00000000000..3137c7d0b07
--- /dev/null
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/JFlexTask.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.gradle.plugins.regenerate;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.GradleException;
+import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.provider.Property;
+import org.gradle.api.tasks.InputFile;
+import org.gradle.api.tasks.Internal;
+import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.TaskAction;
+
+public abstract class JFlexTask extends DefaultTask {
+ @InputFile
+ public abstract RegularFileProperty getJflexFile();
+
+ @InputFile
+ public abstract RegularFileProperty getSkeleton();
+
+ @Internal
+ public abstract Property<String> getHeapSize();
+
+ @OutputFile
+ public abstract RegularFileProperty getGeneratedFile();
+
+ // This is marked internal instead of @Classpath because classpath
+ // files are then added as inputs and end up in the regenerate's
+ // extension checksums file.
+ @Internal
+ public abstract ConfigurableFileCollection getClasspath();
+
+ public JFlexTask() {
+ getGeneratedFile()
+ .convention(
+ getJflexFile()
+ .map(
+ f ->
+ getProject()
+ .getLayout()
+ .getProjectDirectory()
+ .file(f.getAsFile().getAbsolutePath().replace(".jflex", ".java"))));
+ }
+
+ @TaskAction
+ public void generate() {
+ var project = getProject();
+ var logger = getLogger();
+
+ var jflexFile = getJflexFile().get().getAsFile();
+ if (!jflexFile.exists()) {
+ throw new GradleException("JFlex file does not exist: " + jflexFile);
+ }
+
+ final File target = getGeneratedFile().get().getAsFile();
+
+ logger.lifecycle("Recompiling JFlex: {}", project.relativePath(jflexFile));
+
+ LuceneBuildGlobalsExtension buildGlobals =
+ project.getExtensions().getByType(LuceneBuildGlobalsExtension.class);
+
+ buildGlobals
+ .getExecOps()
+ .javaexec(
+ spec -> {
+ spec.setIgnoreExitValue(false);
+
+ spec.setClasspath(getClasspath());
+ spec.getMainClass().set("jflex.Main");
+
+ spec.args("-nobak", "--quiet", "--encoding", "UTF-8");
+
+ if (getHeapSize().isPresent()) {
+ spec.setMaxHeapSize(getHeapSize().get());
+ }
+
+ if (getSkeleton().isPresent()) {
+ spec.args("--skel", getSkeleton().get().getAsFile().getAbsolutePath());
+ }
+
+ spec.args(
+ "-d", target.getParentFile().getAbsolutePath(), jflexFile.getAbsolutePath());
+ });
+
+ try {
+ var text = Files.readString(target.toPath());
+
+ // fix invalid SuppressWarnings from jflex, so that it works with javac.
+ // we also need to suppress 'unused' because of dead code, so it passes ecjLint
+ // you can't "stack" SuppressWarnings annotations, jflex adds its own, this is the only way
+ // https://github.com/jflex-de/jflex/issues/762
+ text =
+ text.replace(
+ "SuppressWarnings(\"fallthrough\")",
+ "SuppressWarnings({\"fallthrough\",\"unused\"})");
+
+ // Seems like a bug in gjf. https://github.com/google/google-java-format/issues/1260
+ text = text.replace("/* Break so we don't hit fall-through warning: */", "");
+
+ Files.writeString(target.toPath(), text);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/JavaCCTask.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/JavaCCTask.java
new file mode 100644
index 00000000000..cce5c8276a2
--- /dev/null
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/JavaCCTask.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.gradle.plugins.regenerate;
+
+import groovy.lang.Closure;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Consumer;
+import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.GradleException;
+import org.gradle.api.Project;
+import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.CopySpec;
+import org.gradle.api.file.FileTree;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.tasks.InputFile;
+import org.gradle.api.tasks.Internal;
+import org.gradle.api.tasks.OutputFiles;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.process.ExecResult;
+import org.gradle.process.JavaExecSpec;
+
+public abstract class JavaCCTask extends DefaultTask {
+ /** Apply closures to all generated files before they're copied back to mainline code. */
+ // A subtle bug here is that this makes it not an input... should be a list of replacements
+ // instead?
+ private final List<Consumer<FileTree>> afterGenerate = new ArrayList<>();
+
+ @InputFile
+ public abstract RegularFileProperty getJavaccFile();
+
+ // This is marked internal instead of @Classpath because classpath
+ // files are then added as inputs and end up in the regenerate's
+ // extension checksums file.
+ @Internal
+ public abstract ConfigurableFileCollection getClasspath();
+
+ public void afterGenerate(Consumer<FileTree> action) {
+ this.afterGenerate.add(action);
+ }
+
+ public void afterGenerate(Closure<?> action) {
+ afterGenerate(action::call);
+ }
+
+ @OutputFiles
+ public List<File> getGeneratedSources() {
+ var javaccFile = getJavaccFile().get().getAsFile();
+ File baseDir = javaccFile.getParentFile();
+ String baseName = javaccFile.getName().replace(".jj", "");
+
+ Project p = getProject();
+ return Arrays.asList(
+ p.file(new File(baseDir, baseName + ".java")),
+ p.file(new File(baseDir, baseName + "Constants.java")),
+ p.file(new File(baseDir, baseName + "TokenManager.java")),
+ p.file(new File(baseDir, "ParseException.java")),
+ p.file(new File(baseDir, "Token.java")),
+ p.file(new File(baseDir, "TokenMgrError.java")));
+ }
+
+ @TaskAction
+ public void generate() {
+ var javaccFile = getJavaccFile().get().getAsFile();
+ if (!javaccFile.exists()) {
+ throw new GradleException("Input file does not exist: " + javaccFile);
+ }
+
+ var logger = getLogger();
+ var project = getProject();
+
+ // Run JavaCC into a temporary dir so we can post-process & know exactly what's generated
+ File tempDir = getTemporaryDir();
+ if (!tempDir.exists() && !tempDir.mkdirs()) {
+ throw new GradleException("Unable to create temporary directory: " + tempDir);
+ }
+
+ // Run javacc generation into temporary folder so that we know all the generated files
+ // and can post-process them easily.
+ LuceneBuildGlobalsExtension buildGlobals =
+ project.getExtensions().getByType(LuceneBuildGlobalsExtension.class);
+ buildGlobals
+ .getFileOps()
+ .delete(spec -> spec.delete(project.fileTree(tempDir, tree -> tree.include("**/*.java"))));
+
+ File targetDir = javaccFile.getParentFile();
+ logger.lifecycle(
+ "Recompiling JavaCC: {}", project.getRootDir().toPath().relativize(javaccFile.toPath()));
+
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ ExecResult result =
+ buildGlobals
+ .getExecOps()
+ .javaexec(
+ (JavaExecSpec spec) -> {
+ spec.setClasspath(getClasspath());
+
+ spec.setIgnoreExitValue(true);
+ spec.setStandardOutput(output);
+ spec.setErrorOutput(output);
+
+ spec.getMainClass().set("org.javacc.parser.Main");
+ spec.args(
+ "-OUTPUT_DIRECTORY=" + tempDir.getAbsolutePath(),
+ javaccFile.getAbsolutePath());
+ });
+
+ if (result.getExitValue() != 0) {
+ throw new GradleException(
+ "JavaCC failed to compile "
+ + javaccFile
+ + ", here is the compilation output:\n"
+ + output);
+ }
+
+ String outString = output.toString(Charset.defaultCharset());
+ if (outString.contains("Warning:")) {
+ throw new GradleException(
+ "JavaCC emitted warnings for "
+ + javaccFile
+ + ", here is the compilation output:\n"
+ + output);
+ }
+
+ // Apply custom modifications
+ FileTree generatedFiles = project.fileTree(tempDir);
+ for (Consumer<FileTree> action : afterGenerate) {
+ action.accept(generatedFiles);
+ }
+
+ // Copy back to mainline sources (excluding CharStream.java)
+ buildGlobals
+ .getFileOps()
+ .copy(
+ (CopySpec spec) -> {
+ spec.from(tempDir);
+ spec.into(targetDir);
+ spec.exclude("CharStream.java");
+ });
+ }
+}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/RegenerateTasksSupportPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/RegenerateTasksSupportPlugin.java
index 407cfa5b8d3..41d557819ca 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/RegenerateTasksSupportPlugin.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/RegenerateTasksSupportPlugin.java
@@ -44,8 +44,8 @@ import org.gradle.api.file.FileCollection;
import org.gradle.api.logging.LogLevel;
import org.gradle.api.provider.Provider;
import org.gradle.api.tasks.Delete;
+import org.gradle.api.tasks.TaskCollection;
import org.gradle.api.tasks.TaskContainer;
-import org.gradle.api.tasks.TaskProvider;
/**
* Adds support for tasks that regenerate (versioned) sources within Lucene.
@@ -84,10 +84,38 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {
@Override
public void apply(Project rootProject) {
applicableToRootProjectOnly(rootProject);
- rootProject.getAllprojects().forEach(RegenerateTasksSupportPlugin::applyRegeneratePlugin);
+
+ var regenTasks =
+ rootProject.getAllprojects().stream()
+ .map(RegenerateTasksSupportPlugin::applyRegeneratePlugin)
+ .toList();
+
+ // wait until after the evaluation is completed to give tasks some time to configure
+ // extension properties, then process these tasks that have the regenerate task extension
+ // configured.
+ rootProject
+ .getGradle()
+ .projectsEvaluated(
+ _ -> {
+ for (var taskCollection : regenTasks) {
+ for (Task task : taskCollection.stream().toList()) {
+ configureRegenerateTask(task);
+ }
+ }
+ });
+
+ // Nearly all regeneration tasks touch input sources, which are inputs to checkLicenses
+ // so schedule this globally, once.
+ rootProject
+ .getTasks()
+ .matching(task -> task.getName().equals(CheckLicensesPlugin.CHECK_LICENSES_TASK))
+ .configureEach(
+ task -> {
+ task.mustRunAfter(regenTasks);
+ });
}
- private static void applyRegeneratePlugin(Project project) {
+ private static TaskCollection<Task> applyRegeneratePlugin(Project project) {
TaskContainer tasks = project.getTasks();
var regenerateTasks =
@@ -112,18 +140,6 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {
.addAll(Set.of("args", "executable", "ignoreExitValue"));
});
- // wait until after the evaluation is completed to give tasks some time to configure
- // extension properties, then process these tasks. I don't
- // think this can be done reasonably well based on lazy properties alone.
- project.afterEvaluate(
- _ -> {
- for (String name : project.getTasks().getNames()) {
- if (REGEN_TASK_PATTERN.matcher(name).matches()) {
- configureRegenerateTask(project, regenerateTasks.named(name));
- }
- }
- });
-
// register a single task to invoke all regeneration tasks.
tasks.register(
REGEN_TASK_NAME,
@@ -141,23 +157,14 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {
task.delete(project.fileTree("src/generated/checksums", cfg -> cfg.include("*.json")));
});
- // Nearly all regeneration tasks touch input sources, which are inputs to checkLicenses
- // so schedule this globally, once.
- project
- .getRootProject()
- .getTasks()
- .matching(task -> task.getName().equals(CheckLicensesPlugin.CHECK_LICENSES_TASK))
- .configureEach(
- task -> {
- task.mustRunAfter(regenerateTasks);
- });
+ return regenerateTasks;
}
- private static void configureRegenerateTask(
- Project project, TaskProvider<? extends Task> taskProvider) {
+ private static void configureRegenerateTask(Task delegate) {
+ var project = delegate.getProject();
var tasks = project.getTasks();
- String taskName = taskProvider.getName();
- String taskPrefix = taskProvider.getName();
+ String taskName = delegate.getName();
+ String taskPrefix = delegate.getName();
Path checksumsFile = project.file("src/generated/checksums/" + taskPrefix + ".json").toPath();
@@ -166,8 +173,6 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {
tasks.register(
taskPrefix + "ChecksumSave",
t -> {
- var delegate = taskProvider.get();
-
// Check delegate's sanity.
if (delegate.getGroup() != null && !delegate.getGroup().isBlank()) {
throw new GradleException(
@@ -208,13 +213,8 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {
});
});
- // TODO: so this entire block could/should reside inside checksumSaveTask.register() and be lazy
- // but gradle throws an 'invalid context' exception on any task.configureEach within this block.
- // I don't know how to do implement this without eager instantiation of tasks, sorry.
+ // Configure task dependencies and wiring of after/before.
{
- Task t = checksumSaveTask.get();
- Task delegate = taskProvider.get();
-
RegenerateTaskExtension regenerateExt =
delegate.getExtensions().getByType(RegenerateTaskExtension.class);
@@ -227,7 +227,7 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {
.matching(task -> taskNames.contains(task.getName()))
.configureEach(
other -> {
- other.mustRunAfter(t);
+ other.mustRunAfter(checksumSaveTask);
});
}
@@ -235,7 +235,7 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {
// checksums are saved but after the source task ran).
{
var taskNames = regenerateExt.getFollowedUpBy().get();
- t.dependsOn(taskNames);
+ checksumSaveTask.configure(t -> t.dependsOn(taskNames));
project
.getTasks()
.matching(task -> taskNames.contains(task.getName()))
@@ -273,18 +273,13 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {
}
}
- taskProvider.configure(
- task -> {
- task.finalizedBy(checksumSaveTask);
- });
+ delegate.finalizedBy(checksumSaveTask);
// Configure checksum check task (verification).
var checksumCheckTask =
tasks.register(
taskName + "ChecksumCheck",
t -> {
- var delegate = taskProvider.get();
-
// if we're regenerating, run any checks after the save.
t.mustRunAfter(checksumSaveTask);
diff --git a/build.gradle b/build.gradle
index bb541ec3399..d95183a0642 100644
--- a/build.gradle
+++ b/build.gradle
@@ -17,21 +17,7 @@
plugins {
id "base"
-
- id "lucene.root-project.check-environment"
id "lucene.root-project.setup"
-
- // TODO: move to java.
- id 'regenerate.forUtil'
- id 'regenerate.antlr'
- id 'regenerate.icu'
- id 'regenerate.javacc'
- id 'regenerate.jflex'
- id 'regenerate.kuromoji'
- id 'regenerate.nori'
- id 'regenerate.unicode-test-classes'
- id 'regenerate.moman'
- id 'regenerate.snowball'
}
description = 'Grandparent project for Apache Lucene Core'
diff --git a/gradle/ide/eclipse/run-solr-cloud.launch b/gradle/ide/eclipse/run-solr-cloud.launch
deleted file mode 100644
index 4f8399a5bc6..00000000000
--- a/gradle/ide/eclipse/run-solr-cloud.launch
+++ /dev/null
@@ -1,20 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
- <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
- <listEntry value="/@ECLIPSEPROJECTNAME@"/>
- </listAttribute>
- <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
- <listEntry value="4"/>
- </listAttribute>
- <booleanAttribute key="org.eclipse.jdt.launching.ATTR_USE_START_ON_FIRST_THREAD" value="true"/>
- <listAttribute key="org.eclipse.jdt.launching.CLASSPATH">
- <listEntry value="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <runtimeClasspathEntry containerPath="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8" javaProject="@ECLIPSEPROJECTNAME@" path="1" type="4"/> "/>
- <listEntry value="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <runtimeClasspathEntry id="org.eclipse.jdt.launching.classpathentry.defaultClasspath"> <memento exportedEntriesOnly="false" project="@ECLIPSEPROJECTNAME@"/> </runtimeClasspathEntry> "/>
- <listEntry value="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <runtimeClasspathEntry internalArchive="/@ECLIPSEPROJECTNAME@/solr/server/start.jar" path="3" type="2"/> "/>
- </listAttribute>
- <booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>
- <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.eclipse.jetty.start.Main"/>
- <stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="--module=http"/>
- <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@ECLIPSEPROJECTNAME@"/>
- <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xms512m -Djetty.port=8983 -DSTOP.PORT=7983 -DSTOP.KEY=solrrock -Dlog4j.configurationFile="file:${workspace_loc:@ECLIPSEPROJECTNAME@}/solr/server/resources/log4j2.xml" -Djetty.home="${workspace_loc:@ECLIPSEPROJECTNAME@}/eclipse-build/solr-server" -Djetty.base="${workspace_loc:@ECLIPSEPROJECTNAME@}/eclipse-build/solr-server" [...]
-</launchConfiguration>
diff --git a/gradle/ide/eclipse/run-solr.launch b/gradle/ide/eclipse/run-solr.launch
deleted file mode 100644
index 4d6b033a405..00000000000
--- a/gradle/ide/eclipse/run-solr.launch
+++ /dev/null
@@ -1,20 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
- <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
- <listEntry value="/@ECLIPSEPROJECTNAME@"/>
- </listAttribute>
- <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
- <listEntry value="4"/>
- </listAttribute>
- <booleanAttribute key="org.eclipse.jdt.launching.ATTR_USE_START_ON_FIRST_THREAD" value="true"/>
- <listAttribute key="org.eclipse.jdt.launching.CLASSPATH">
- <listEntry value="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <runtimeClasspathEntry containerPath="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8" javaProject="@ECLIPSEPROJECTNAME@" path="1" type="4"/> "/>
- <listEntry value="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <runtimeClasspathEntry id="org.eclipse.jdt.launching.classpathentry.defaultClasspath"> <memento exportedEntriesOnly="false" project="@ECLIPSEPROJECTNAME@"/> </runtimeClasspathEntry> "/>
- <listEntry value="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <runtimeClasspathEntry internalArchive="/@ECLIPSEPROJECTNAME@/solr/server/start.jar" path="3" type="2"/> "/>
- </listAttribute>
- <booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>
- <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.eclipse.jetty.start.Main"/>
- <stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="--module=http"/>
- <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@ECLIPSEPROJECTNAME@"/>
- <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xms512m -Djetty.port=8983 -DSTOP.PORT=7983 -DSTOP.KEY=solrrock -Dlog4j.configurationFile="file:${workspace_loc:@ECLIPSEPROJECTNAME@}/solr/server/resources/log4j2.xml" -Djetty.home="${workspace_loc:@ECLIPSEPROJECTNAME@}/eclipse-build/solr-server" -Djetty.base="${workspace_loc:@ECLIPSEPROJECTNAME@}/eclipse-build/solr-server"  [...]
-</launchConfiguration>
diff --git a/gradle/ide/eclipse/run-test-cases.launch b/gradle/ide/eclipse/run-test-cases.launch
deleted file mode 100644
index 713628cebab..00000000000
--- a/gradle/ide/eclipse/run-test-cases.launch
+++ /dev/null
@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<launchConfiguration type="org.eclipse.jdt.junit.launchconfig">
- <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
- <listEntry value="/@ECLIPSEPROJECTNAME@"/>
- </listAttribute>
- <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
- <listEntry value="4"/>
- </listAttribute>
- <stringAttribute key="org.eclipse.jdt.junit.CONTAINER" value="=@ECLIPSEPROJECTNAME@"/>
- <booleanAttribute key="org.eclipse.jdt.junit.KEEPRUNNING_ATTR" value="false"/>
- <stringAttribute key="org.eclipse.jdt.junit.TESTNAME" value=""/>
- <stringAttribute key="org.eclipse.jdt.junit.TEST_KIND" value="org.eclipse.jdt.junit.loader.junit4"/>
- <booleanAttribute key="org.eclipse.jdt.launching.ATTR_USE_START_ON_FIRST_THREAD" value="true"/>
- <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value=""/>
- <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@ECLIPSEPROJECTNAME@"/>
- <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-ea"/>
-</launchConfiguration>
diff --git a/gradle/regenerate/moman/gen_BulkOperation.py b/gradle/regenerate/packed/gen_BulkOperation.py
similarity index 100%
rename from gradle/regenerate/moman/gen_BulkOperation.py
rename to gradle/regenerate/packed/gen_BulkOperation.py
diff --git a/gradle/regenerate/moman/gen_Packed64SingleBlock.py b/gradle/regenerate/packed/gen_Packed64SingleBlock.py
similarity index 100%
rename from gradle/regenerate/moman/gen_Packed64SingleBlock.py
rename to gradle/regenerate/packed/gen_Packed64SingleBlock.py
diff --git a/lucene/analysis/common/build.gradle b/lucene/analysis/common/build.gradle
index 59c95921e11..480f7b45e97 100644
--- a/lucene/analysis/common/build.gradle
+++ b/lucene/analysis/common/build.gradle
@@ -15,7 +15,9 @@
* limitations under the License.
*/
-
+import org.apache.lucene.gradle.plugins.misc.QuietExec
+import de.undercouch.gradle.tasks.download.Download
+import org.apache.tools.ant.taskdefs.condition.Os
import com.carrotsearch.gradle.buildinfra.buildoptions.BuildOption
import java.nio.file.Files
@@ -27,6 +29,105 @@ dependencies {
moduleTestImplementation project(':lucene:test-framework')
}
+//
+// Regenerate snowball resources
+//
+
+def resources = rootProject.file("gradle/regenerate/snowball")
+
+// git commit hash of source code https://github.com/snowballstem/snowball/
+def snowballStemmerCommit = "34f3612e5e8c48975243bc2e87561abdac5aa9bb"
+// git commit hash of stopwords https://github.com/snowballstem/snowball-website
+def snowballWebsiteCommit = "424fd1f75044160bb4a92f1daf08cce618459374"
+
+def snowballWorkDir = file("${buildDir}/snowball")
+
+def snowballStemmerDir = file("${snowballWorkDir}/stemmers-${snowballStemmerCommit}")
+def snowballWebsiteDir = file("${snowballWorkDir}/website-${snowballWebsiteCommit}")
+
+def snowballScript = file("${resources}/snowball.sh")
+
+def unpackFromZip = { zipFile, targetDir ->
+ buildGlobals.fileOps.sync({SyncSpec spec ->
+ spec.from(zipTree(zipFile), {
+ eachFile { fcd ->
+ fcd.relativePath = new RelativePath(true, fcd.relativePath.segments.drop(1))
+ }
+ })
+ spec.into targetDir
+ })
+}
+
+// downloads snowball stemmers (or use a cached copy)
+def downloadSnowballStemmers = tasks.register("downloadSnowballStemmers", Download, {
+ src "https://github.com/snowballstem/snowball/archive/${snowballStemmerCommit}.zip"
+ dest file("${snowballStemmerDir}.zip")
+ overwrite false
+ tempAndMove true
+
+ doLast {
+ unpackFromZip(dest, snowballStemmerDir)
+ }
+})
+
+// downloads snowball website (or use a cached copy)
+def downloadSnowballWebsite = tasks.register("downloadSnowballWebsite", Download, {
+ src "https://github.com/snowballstem/snowball-website/archive/${snowballWebsiteCommit}.zip"
+ def snowballWebsiteZip = file("${snowballWebsiteDir}.zip")
+ dest snowballWebsiteZip
+ overwrite false
+ tempAndMove true
+
+ doLast {
+ unpackFromZip(snowballWebsiteZip, snowballWebsiteDir)
+ }
+})
+
+// runs a shell script to regenerate stemmers, base stemming subclasses, test data, and stopwords.
+tasks.register("regenerateSnowball", QuietExec, {
+ description = "Regenerate snowball stemmers."
+
+ dependsOn downloadSnowballStemmers
+ dependsOn downloadSnowballWebsite
+
+ regenerate {
+ ifSkippedAlsoSkip = [
+ downloadSnowballStemmers.name,
+ downloadSnowballWebsite.name
+ ]
+ }
+
+ inputs.files fileTree(
+ dir: "src/java/org/tartarus/snowball",
+ include: [
+ "Among.java",
+ "SnowballStemmer.java",
+ "SnowballProgram.java",
+ "ext/*Stemmer.java"
+ ])
+
+ inputs.files fileTree(
+ dir: "src/resources/org/apache/lucene/analysis/snowball",
+ include: "*_stop.txt")
+
+ // Don't bother on Windows, won't work.
+ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+ doFirst {
+ // Just emit a big fat error message. Fail the build so that checksums are not regenerated.
+ throw new GradleException("Snowball generation does not work on Windows (bash must be available).")
+ }
+ }
+
+ executable "bash"
+ args = [
+ snowballScript,
+ snowballStemmerDir,
+ snowballWebsiteDir,
+ projectDir
+ ]
+})
+
+
// Enable Hunspell tests against LibreOffice/ Woorm dictionaries. We pull
// these dictionaries dynamically from git of each respective project. To keep
// things consistent across pull requests/ re-runs, we use a fixed git commit
diff --git a/lucene/analysis/icu/build.gradle b/lucene/analysis/icu/build.gradle
index 3bcde2832f4..d307ec65520 100644
--- a/lucene/analysis/icu/build.gradle
+++ b/lucene/analysis/icu/build.gradle
@@ -15,7 +15,8 @@
* limitations under the License.
*/
-
+import org.apache.tools.ant.taskdefs.condition.Os
+import java.nio.file.Files
description = 'Analysis integration with ICU (International Components for Unicode)'
@@ -27,3 +28,286 @@ dependencies {
moduleTestImplementation project(':lucene:test-framework')
}
+
+/*
+ * Regenerates ICU-related data files in this and other projects.
+ *
+ * The icu4c version must match exactly the icu4j version in libs.versions.toml.
+ * The one on your system is probably different. This script will attempt to
+ * download and compile a matching icu4c version automatically.
+ */
+
+def resources = rootProject.file("gradle/regenerate/icu")
+
+def setupIcuDependencies = { Project proj ->
+ // Configure different icu4j dependencies.
+ proj.configurations {
+ icu_current
+ groovy
+ }
+
+ proj.dependencies {
+ icu_current deps.icu4j
+ groovy deps.groovy
+ }
+}
+
+def icu4jVersion = deps.icu4j.get().version
+
+setupIcuDependencies(project)
+def utr30DataDir = file("src/data/utr30")
+
+def icuBuildDir = file("${buildDir}/icu")
+def icuBinDir
+def gennorm
+def icupkg
+
+if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+ icuBinDir = file("${icuBuildDir}/bin64")
+ gennorm = file("${icuBinDir}/gennorm2.exe")
+ icupkg = file("${icuBinDir}/icupkg.exe")
+} else {
+ icuBinDir = file("${icuBuildDir}/icu/source/bin")
+ gennorm = file("${icuBinDir}/gennorm2")
+ icupkg = file("${icuBinDir}/icupkg")
+}
+
+def icuCompileTask = Os.isFamily(Os.FAMILY_WINDOWS) ? "compileIcuWindows" : "compileIcuLinux"
+
+tasks.register("regenerateUtr30DataFiles", {
+ description = "Regenerate utf30 norm files using ICU"
+
+ def icuConfig = configurations.icu_current
+
+ dependsOn icuConfig
+ dependsOn icuCompileTask
+
+ regenerate {
+ ifSkippedAlsoSkip = [icuCompileTask]
+ }
+
+ // May be undefined yet, so use a provider.
+ dependsOn { sourceSets.tools.runtimeClasspath }
+
+ // gennorm generates file order-dependent output, so make it constant here.
+ def inputFiles = fileTree(dir: utr30DataDir, include: "*.txt")
+ .asList().toSorted(Comparator.comparing { File f -> f.name })
+ def outputFile = file("src/resources/org/apache/lucene/analysis/icu/utr30.nrm")
+
+ inputs.files inputFiles
+ inputs.property "icuConfig", icu4jVersion
+ outputs.file outputFile
+
+ doFirst {
+ // all these steps must be done sequentially: it's a pipeline resulting in utr30.nrm
+ buildGlobals.execOps.javaexec {
+ mainClass = "org.apache.lucene.analysis.icu.GenerateUTR30DataFiles"
+ classpath = sourceSets.tools.runtimeClasspath
+
+ ignoreExitValue = false
+ workingDir utr30DataDir
+ args = [
+ "release-${icu4jVersion.replace(".", "-")}"
+ ]
+ }
+
+ buildGlobals.quietExec(it, {
+ executable gennorm
+ args = [
+ "-v",
+ "-s",
+ utr30DataDir,
+ "-o",
+ "${buildDir}/utr30.tmp",
+ *(inputFiles.collect { it.name })
+ ]
+ })
+
+ buildGlobals.quietExec(it, {
+ executable icupkg
+ args = [
+ "-tb",
+ "${buildDir}/utr30.tmp",
+ outputFile
+ ]
+ })
+ }
+})
+
+tasks.register("regenerateRbbi", {
+ description = "Regenerate rbbi rules using ICU"
+
+ def icuConfig = configurations.icu_current
+
+ dependsOn icuConfig
+
+ // May be undefined yet, so use a provider.
+ dependsOn { sourceSets.tools.runtimeClasspath }
+
+ def sourceDir = file("src/data/uax29")
+ def targetDir = file("src/resources/org/apache/lucene/analysis/icu/segmentation")
+
+ inputs.files fileTree(dir: sourceDir, include: "*.rbbi")
+ inputs.property "icuConfig", icu4jVersion
+ outputs.files fileTree(dir: targetDir, include: "*.brk")
+
+ doFirst {
+ buildGlobals.execOps.javaexec {
+ mainClass = "org.apache.lucene.analysis.icu.RBBIRuleCompiler"
+ classpath = sourceSets.tools.runtimeClasspath
+
+ ignoreExitValue = false
+ enableAssertions true
+ args = [sourceDir, targetDir]
+ }
+ }
+})
+
+tasks.register("compileIcuWindows", {
+ doFirst {
+ def v = icu4jVersion
+ def icuBinZip = file("${icuBuildDir}/icu4c-${v.replace(".", "_")}.zip")
+ if (!icuBinZip.exists()) {
+ icuBuildDir.mkdirs()
+
+ // Download binaries matching icu4j version in version.props
+ def src = URI.create("https://github.com/unicode-org/icu/releases/download/release-${v.replace(".", "-")}/icu4c-${v.replace(".", "_")}-Win64-MSVC2019.zip")
+
+ logger.lifecycle("Trying to download binary ICU version: ${v} from:\n ${src}")
+ Files.write(icuBinZip.toPath(), src.toURL().openStream().bytes)
+ logger.lifecycle("Downloaded ${icuBinZip.size()} bytes.")
+ }
+
+ // Unzip.
+ buildGlobals.fileOps.copy({
+ into icuBuildDir
+ from zipTree(icuBinZip)
+ })
+ }
+})
+
+tasks.register("compileIcuLinux", {
+ doFirst {
+ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+ throw new GradleException("ICU compilation not supported on Windows.")
+ }
+
+ def v = icu4jVersion
+ def icuSrcTgz = file("${icuBuildDir}/icu4c-${v.replace(".", "_")}-src.tgz")
+
+ // Download sources for version matching icu4j version in version.props
+ if (!icuSrcTgz.exists()) {
+ icuBuildDir.mkdirs()
+ def src = URI.create("https://github.com/unicode-org/icu/releases/download/release-${v.replace(".", "-")}/icu4c-${v.replace(".", "_")}-src.tgz")
+ logger.lifecycle("Trying to download and compile ICU version: ${v} from:\n ${src}")
+ Files.write(icuSrcTgz.toPath(), src.toURL().openStream().bytes)
+ logger.lifecycle("Downloaded ${icuSrcTgz.size()} bytes.")
+ }
+
+ def icuSrcDir = file("${icuBuildDir}/icu/source")
+ buildGlobals.fileOps.delete(icuSrcDir)
+
+ // Extract the tgz
+ buildGlobals.quietExec(it, {
+ executable "tar"
+ workingDir icuBuildDir
+ args = [
+ "-zxf",
+ icuSrcTgz
+ ]
+ })
+
+ // Compile: (cd icu/source && ./configure --prefix=$(pwd) --enable-rpath && make -j4)
+ buildGlobals.quietExec(it, {
+ executable "sh"
+
+ workingDir icuSrcDir
+ environment("CFLAGS", "-O0")
+ environment("CXXFLAGS", "-O0")
+ args = [
+ "configure",
+ "--prefix=${icuSrcDir}",
+ "--enable-rpath"
+ ]
+ })
+
+ buildGlobals.quietExec(it, {
+ executable "make"
+ workingDir icuSrcDir
+ args = [
+ "-j${buildOptions['tests.jvms'].get()}"
+ ]
+ })
+
+ // Test that the binaries work: derb -V
+ logger.lifecycle("Compiled ICU, checking...")
+ buildGlobals.quietExec(it, {
+ executable "./derb"
+ workingDir icuBinDir
+ args = ["-V"]
+ })
+ }
+})
+
+configure(project(":lucene:analysis:common")) {
+ setupIcuDependencies(project)
+
+ tasks.register("regenerateUnicodeProps", {
+ def icuConfig = configurations.icu_current
+ def groovyConfig = configurations.groovy
+
+ def outputFile = file("src/java/org/apache/lucene/analysis/util/UnicodeProps.java")
+
+ description = "Regenerate ${outputFile} using ICU"
+
+ dependsOn icuConfig
+
+ inputs.property "icuConfig", icu4jVersion
+ outputs.file outputFile
+
+ doFirst {
+ buildGlobals.execOps.javaexec({ JavaExecSpec spec ->
+ spec.mainClass = "groovy.lang.GroovyShell"
+ spec.classpath(icuConfig, groovyConfig)
+
+ spec.args([
+ "--encoding",
+ "UTF-8",
+ file("${resources}/GenerateUnicodeProps.groovy").toString(),
+ outputFile.toString()
+ ])
+ })
+ }
+ })
+}
+
+configure(project(":lucene:core")) {
+ setupIcuDependencies(project)
+
+ tasks.register("generateUnicodeProps", {
+ def icuConfig = configurations.icu_current
+ def groovyConfig = configurations.groovy
+ def outputFile = file("src/java/org/apache/lucene/util/automaton/CaseFolding.java")
+
+ description = "Regenerate ${outputFile} using ICU"
+
+ dependsOn icuConfig
+
+ inputs.property "icuConfig", icu4jVersion
+ outputs.file outputFile
+
+ doFirst {
+ buildGlobals.execOps.javaexec({
+ mainClass = "groovy.lang.GroovyShell"
+ classpath icuConfig, groovyConfig
+
+ args = [
+ "--encoding",
+ "UTF-8",
+ file("${resources}/GenerateCaseFolding.groovy"),
+ outputFile
+ ]
+ })
+ }
+ })
+}
diff --git a/lucene/analysis/kuromoji/build.gradle b/lucene/analysis/kuromoji/build.gradle
index 9119a5c50fa..fdbd8a6a6db 100644
--- a/lucene/analysis/kuromoji/build.gradle
+++ b/lucene/analysis/kuromoji/build.gradle
@@ -1,3 +1,6 @@
+import de.undercouch.gradle.tasks.download.Download
+import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -25,3 +28,126 @@ dependencies {
moduleTestImplementation project(':lucene:test-framework')
}
+
+//
+// This configures regeneration of Kuromoji dictionaries.
+//
+
+static void recompileDictionary(Project project, String dictionaryName, Closure closure) {
+ project.getExtensions().getByType(LuceneBuildGlobalsExtension).execOps.javaexec {
+ mainClass = "org.apache.lucene.analysis.ja.dict.DictionaryBuilder"
+ classpath = project.sourceSets.main.runtimeClasspath
+
+ jvmArgs '-Xmx1G'
+
+ with closure
+ }
+ project.logger.lifecycle("Automaton regenerated from dictionary: ${dictionaryName}")
+}
+
+def targetDir = project.file("src/resources")
+
+def mecabDictionaryName = "mecab-ipadic-2.7.0-20070801"
+// original source: "https://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/${mecabDictionaryName}.tar.gz"
+def mecabDictionarySource = "https://s3.amazonaws.com/lucene-testdata/mecab/${mecabDictionaryName}.tar.gz"
+
+def compileMecab = tasks.register("compileMecab", Download, {
+ dependsOn sourceSets.main.runtimeClasspath
+
+ def dictionaryFile = file("${buildDir}/generate/${mecabDictionaryName}.tar.gz")
+ def unpackedDir = file("${buildDir}/generate/${mecabDictionaryName}")
+
+ src mecabDictionarySource
+ dest dictionaryFile
+ onlyIfModified true
+
+ doLast {
+ buildGlobals.fileOps.delete(sourceSets.main.resources.srcDirs.collect {File location ->
+ fileTree(dir: location, include: "org/apache/lucene/analysis/ja/dict/*.dat")
+ })
+
+ // Unpack the downloaded archive.
+ delete unpackedDir
+ ant.untar(src: dictionaryFile, dest: unpackedDir, compression: "gzip") {
+ ant.cutdirsmapper(dirs: "1")
+ }
+
+ // Apply patch via local git.
+ buildGlobals.quietExec(it, {
+ workingDir = unpackedDir
+ executable "git"
+ args += [
+ "apply",
+ project.file("src/tools/patches/Noun.proper.csv.patch").absolutePath
+ ]
+ })
+
+ // Compile the dictionary
+ recompileDictionary(project, mecabDictionaryName, {
+ args += [
+ "ipadic",
+ unpackedDir,
+ targetDir,
+ "euc-jp",
+ false
+ ]
+ })
+ }
+})
+
+tasks.register("regenerateDictionariesFromMecab", {
+ description = "Regenerate dictionaries from Mecab data"
+
+ dependsOn compileMecab
+
+ inputs.property("dictionarySource", mecabDictionarySource)
+ outputs.files(fileTree(dir: "src/resources/org/apache/lucene/analysis/ja/dict", include: "*.dat"))
+
+ regenerate {
+ // No need to run any follow-ups, we generate binary files (not sources).
+ followedUpBy = []
+ ifSkippedAlsoSkip = [compileMecab.name]
+ }
+})
+
+// TODO: this regenerates data files over the same files as mecab, above. This seems odd and inconsistent?
+// https://github.com/apache/lucene/issues/15365
+
+// original source (down): https://rwthaachen.dl.osdn.jp/naist-jdic/53500/mecab-naist-jdic-0.6.3b-20111013.tar.gz
+def naistDictionaryName = "mecab-naist-jdic-0.6.3b-20111013"
+def naistDictionarySource = "https://s3.amazonaws.com/lucene-testdata/mecab/${naistDictionaryName}.tar.gz"
+tasks.register("compileNaist", Download, {
+ description = "Recompile dictionaries from Naist data."
+
+ dependsOn sourceSets.main.runtimeClasspath
+
+ def dictionaryFile = file("${buildDir}/generate/${naistDictionaryName}.tar.gz")
+ def unpackedDir = file("${buildDir}/generate/${naistDictionaryName}")
+
+ src naistDictionarySource
+ dest dictionaryFile
+ onlyIfModified true
+
+ doLast {
+ buildGlobals.fileOps.delete(sourceSets.main.resources.srcDirs.collect {File location ->
+ fileTree(dir: location, include: "org/apache/lucene/analysis/ja/dict/*.dat")
+ })
+
+ // Unpack the downloaded archive.
+ delete unpackedDir
+ ant.untar(src: dictionaryFile, dest: unpackedDir, compression: "gzip") {
+ ant.cutdirsmapper(dirs: "1")
+ }
+
+ // Compile the dictionary
+ recompileDictionary(project, naistDictionaryName, {
+ args += [
+ "ipadic",
+ unpackedDir,
+ targetDir,
+ "euc-jp",
+ false
+ ]
+ })
+ }
+})
diff --git a/lucene/analysis/nori/build.gradle b/lucene/analysis/nori/build.gradle
index b9e82d0c90a..56ded495e6e 100644
--- a/lucene/analysis/nori/build.gradle
+++ b/lucene/analysis/nori/build.gradle
@@ -15,7 +15,8 @@
* limitations under the License.
*/
-
+import de.undercouch.gradle.tasks.download.Download
+import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension
description = 'Korean Morphological Analyzer'
@@ -25,3 +26,76 @@ dependencies {
moduleTestImplementation project(':lucene:test-framework')
}
+
+//
+// This configures regeneration of Nori dictionaries.
+//
+
+static void recompileDictionary(Project project, String dictionaryName, Closure closure) {
+ project.getExtensions().getByType(LuceneBuildGlobalsExtension).execOps.javaexec {
+ mainClass = "org.apache.lucene.analysis.ko.dict.DictionaryBuilder"
+ classpath = project.sourceSets.main.runtimeClasspath
+
+ jvmArgs '-Xmx1G'
+
+ with closure
+ }
+ project.logger.lifecycle("Automaton regenerated from dictionary: ${dictionaryName}")
+}
+
+plugins.withType(JavaPlugin).configureEach {
+ project.ext {
+ targetDir = project.file("src/resources")
+ }
+
+ def dictionaryName = "mecab-ko-dic-2.1.1-20180720"
+ def dictionarySource = "https://s3.amazonaws.com/lucene-testdata/mecab/${dictionaryName}.tar.gz"
+ def compileMecabKo = tasks.register("compileMecabKo", Download, {
+
+ dependsOn sourceSets.main.runtimeClasspath
+
+ def dictionaryFile = file("${buildDir}/generate/${dictionaryName}.tar.gz")
+ def unpackedDir = file("${buildDir}/generate/${dictionaryName}")
+
+ src dictionarySource
+ dest dictionaryFile
+ onlyIfModified true
+
+ doLast {
+ buildGlobals.fileOps.delete(sourceSets.main.resources.srcDirs.collect {File location ->
+ fileTree(dir: location, include: "org/apache/lucene/analysis/ja/dict/*.dat")
+ })
+
+ // Unpack the downloaded archive.
+ delete unpackedDir
+ ant.untar(src: dictionaryFile, dest: unpackedDir, compression: "gzip") {
+ ant.cutdirsmapper(dirs: "1")
+ }
+
+ // Compile the dictionary
+ recompileDictionary(project, dictionaryName, {
+ args += [
+ unpackedDir,
+ targetDir,
+ "utf-8",
+ false
+ ]
+ })
+ }
+ })
+
+ tasks.register("regenerateDictionariesFromMecabKo", {
+ description = "Recompile dictionaries from Mecab-Ko data."
+
+ dependsOn compileMecabKo
+
+ inputs.property("dictionarySource", dictionarySource)
+ outputs.files(fileTree(dir: "src/resources/org/apache/lucene/analysis/ko/dict", include: "*.dat"))
+
+ regenerate {
+ // No need to run any follow-ups, we generate binary files (not sources).
+ followedUpBy = []
+ ifSkippedAlsoSkip = ["compileMecabKo"]
+ }
+ })
+}
diff --git a/lucene/benchmark-jmh/build.gradle b/lucene/benchmark-jmh/build.gradle
index 2a8232c37ce..6f874e410b9 100644
--- a/lucene/benchmark-jmh/build.gradle
+++ b/lucene/benchmark-jmh/build.gradle
@@ -30,7 +30,7 @@ dependencies {
// Exclude JMH-generated files and certain classes that require incubating classes
// from forbiddenapis validation.
-tasks.matching { it.name == "forbiddenApisMain" }.configureEach {
+tasks.named("forbiddenApisMain").configure {
patternSet.exclude(["**/jmh_generated/**"])
}
@@ -62,7 +62,7 @@ tasks.compileJava.configure {
options.javaModuleMainClass.set("org.apache.lucene.benchmark.jmh.Main")
}
-jar.configure { Jar jarTask ->
+tasks.named("jar").configure {
dependsOn configurations.runtimeClasspath
manifest {
diff --git a/lucene/core/build.gradle b/lucene/core/build.gradle
index 2f7f461f71e..ae4f31261ae 100644
--- a/lucene/core/build.gradle
+++ b/lucene/core/build.gradle
@@ -14,8 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+import de.undercouch.gradle.tasks.download.Download
+import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension
+import org.apache.lucene.gradle.plugins.misc.QuietExec
import org.apache.lucene.gradle.plugins.mrjar.LuceneJavaCoreMrjarPlugin
import org.apache.lucene.gradle.plugins.mrjar.MrJarsExtension
+import org.apache.lucene.gradle.plugins.regenerate.JFlexTask
plugins.apply(LuceneJavaCoreMrjarPlugin)
@@ -28,3 +33,342 @@ dependencies {
// Setup api stub generation and multi-release jar support for the following JDKs.
project.getExtensions().getByType(MrJarsExtension).setupFor(Integer.parseInt(deps.versions.minJava.get()))
+
+//
+// Regenerates "ForUtil.java" code used by codecs in this project and in backward-codecs
+//
+
+static void createForUtilRegenerateTask(
+ Project project, String taskName, String javaFile, String pyScript, File genDir) {
+ project.getTasks().register(
+ "regenerate" + taskName, QuietExec, {
+ description = "Regenerate " + javaFile + " using " + pyScript + " in " + genDir
+
+ def genScript = project.file("${genDir}/${pyScript}")
+ def genOutput = project.file("${genDir}/${javaFile}")
+ inputs.file genScript
+ outputs.file genOutput
+
+ workingDir = genDir
+ executable = project.getExtensions().getByType(LuceneBuildGlobalsExtension).externalTool("python3")
+ args = ['-B', genScript.toString()]
+ })
+}
+
+createForUtilRegenerateTask(project, "ForUtil", "ForUtil.java", "gen_ForUtil.py",
+ file("src/java/org/apache/lucene/codecs/lucene104"))
+
+configure(project(":lucene:backward-codecs")) {
+ createForUtilRegenerateTask(project, "ForUtil84", "ForUtil.java", "gen_ForUtil.py",
+ file("src/java/org/apache/lucene/backward_codecs/lucene84"))
+
+ createForUtilRegenerateTask(project, "ForUtil90", "ForUtil.java", "gen_ForUtil.py",
+ file("src/java/org/apache/lucene/backward_codecs/lucene90"))
+
+ createForUtilRegenerateTask(project, "ForUtil99", "ForUtil.java", "gen_ForUtil.py",
+ file("src/java/org/apache/lucene/backward_codecs/lucene99"))
+
+ createForUtilRegenerateTask(project, "ForUtil912", "ForUtil.java", "gen_ForUtil.py",
+ file("src/java/org/apache/lucene/backward_codecs/lucene912"))
+ createForUtilRegenerateTask(project, "ForDeltaUtil912", "ForDeltaUtil.java", "gen_ForDeltaUtil.py",
+ file("src/java/org/apache/lucene/backward_codecs/lucene912"))
+
+ createForUtilRegenerateTask(project, "ForUtil101", "ForUtil.java", "gen_ForUtil.py",
+ file("src/java/org/apache/lucene/backward_codecs/lucene101"))
+ createForUtilRegenerateTask(project, "ForDeltaUtil101", "ForDeltaUtil.java", "gen_ForDeltaUtil.py",
+ file("src/java/org/apache/lucene/backward_codecs/lucene101"))
+
+ createForUtilRegenerateTask(project, "ForUtil103", "ForUtil.java", "gen_ForUtil.py",
+ file("src/java/org/apache/lucene/backward_codecs/lucene103"))
+ createForUtilRegenerateTask(project, "ForDeltaUtil103", "ForDeltaUtil.java", "gen_ForDeltaUtil.py",
+ file("src/java/org/apache/lucene/backward_codecs/lucene103"))
+}
+
+//
+// Regenerate jflex generated sources in this and other projects
+//
+
+def resources = rootProject.file("gradle/regenerate/jflex")
+def skeletonDefault = file("${resources}/skeleton.default.txt")
+def skeletonNoBufferExpansion = file("${resources}/skeleton.disable.buffer.expansion.txt")
+
+configure(project(":lucene:core")) {
+ configurations {
+ jflex
+ }
+
+ dependencies {
+ jflex deps.jflex
+ }
+
+ tasks.register("regenerateStandardTokenizer", JFlexTask, {
+ description = "Regenerate StandardTokenizerImpl.java"
+
+ classpath = configurations.jflex
+
+ jflexFile = file('src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex')
+ skeleton = skeletonNoBufferExpansion
+
+ doLast {
+ ant.replace(
+ file: file('src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java'),
+ encoding: "UTF-8",
+ token: "private static final int ZZ_BUFFERSIZE =",
+ value: "private int ZZ_BUFFERSIZE ="
+ )
+ }
+ })
+}
+
+configure(project(":lucene:analysis:common")) {
+ configurations {
+ jflex
+ }
+
+ dependencies {
+ jflex deps.jflex
+ }
+
+ def regenerateTlds = tasks.register("regenerateTlds", {
+ def tldZones = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
+ def jflexMacro = file("src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex")
+ def tldList = file("src/test/org/apache/lucene/analysis/email/TLDs.txt")
+
+ description = "Regenerate top-level domain jflex macros and tests"
+
+ dependsOn { sourceSets.tools.runtimeClasspath }
+
+ inputs.property "tldZones", tldZones
+ outputs.files jflexMacro, tldList
+
+ doFirst {
+ File tmpJflexMacro = File.createTempFile(jflexMacro.getName(), ".tmp", getTemporaryDir())
+ File tmpTldList = File.createTempFile(tldList.getName(), ".tmp", getTemporaryDir())
+
+ project.extensions.buildGlobals.execOps.javaexec {
+ mainClass = "org.apache.lucene.analysis.standard.GenerateJflexTLDMacros"
+ classpath = sourceSets.tools.runtimeClasspath
+
+ ignoreExitValue = false
+ args = [
+ tldZones,
+ tmpJflexMacro,
+ tmpTldList
+ ]
+ }
+
+ // LUCENE-9926: tldZones is regenerated daily. Compare the generated content (excluding comments) so that
+ // we only update actual output files if non-comments have changed.
+ def contentLines = { File file ->
+ if (file.exists()) {
+ List<String> lines = file.readLines("UTF-8")
+ lines.removeIf { line -> line.isBlank() || line.startsWith("//") }
+ return lines
+ } else {
+ return []
+ }
+ }
+
+ if (contentLines(tmpTldList).equals(contentLines(tldList))) {
+ logger.lifecycle("Generated TLD content identical as before, not updating.")
+ } else {
+ tldList.setBytes tmpTldList.bytes
+ jflexMacro.setBytes tmpJflexMacro.bytes
+ logger.lifecycle("You've regenerated the TLD include file, remember to regenerate UAX29URLEmailTokenizerImpl too.")
+ }
+ }
+ })
+
+ tasks.register("regenerateWikipediaTokenizer", JFlexTask, {
+ description = "Regenerate WikipediaTokenizerImpl.java"
+
+ classpath = configurations.jflex
+
+ jflexFile = file('src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex')
+ skeleton = skeletonDefault
+ })
+
+ tasks.register("regenerateClassicTokenizer", JFlexTask, {
+ description = "Regenerate ClassicTokenizerImpl.java"
+
+ classpath = configurations.jflex
+
+ jflexFile = file('src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.jflex')
+ skeleton = skeletonDefault
+ })
+
+ tasks.register("regenerateUAX29URLEmailTokenizer", JFlexTask, {
+ description = "Regenerate UAX29URLEmailTokenizerImpl.java"
+
+ classpath = configurations.jflex
+
+ // UAX29URLEmailTokenizerImpl.jflex includes: ASCIITLD.jflex
+ // so we make sure it is up to date. Also, make sure the ordering of internal
+ // tasks is enforced (just scheduling top-level wrappers doesn't mean their subtask graphs cannot be reordered).
+ dependsOn regenerateTlds
+
+ jflexFile = file('src/java/org/apache/lucene/analysis/email/UAX29URLEmailTokenizerImpl.jflex')
+ skeleton = skeletonNoBufferExpansion
+ heapSize = "12g"
+
+ // Add included files as inputs.
+ inputs.file file('src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex')
+
+ doFirst {
+ logger.lifecycle("Regenerating UAX29URLEmailTokenizerImpl. This may take a long time (and requires ${heapSize} of memory!).")
+ }
+
+ doLast {
+ ant.replace(
+ file: file('src/java/org/apache/lucene/analysis/email/UAX29URLEmailTokenizerImpl.java'),
+ encoding: "UTF-8",
+ token: "private static final int ZZ_BUFFERSIZE =",
+ value: "private int ZZ_BUFFERSIZE ="
+ )
+ }
+ })
+
+ def generateHTMLCharacterEntities = tasks.register("regenerateHTMLCharacterEntities", QuietExec, {
+ description = "Regenerate HTMLCharacterEntities.jflex"
+
+ def target = file('src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex')
+ def script = file("${resources}/htmlentity.py")
+
+ outputs.files target
+ inputs.file script
+
+ executable = buildGlobals.externalTool("python3")
+ workingDir = target.parentFile
+ args += [
+ "-B",
+ // don't write any bytecode cache
+ script,
+ target
+ ]
+
+ doLast {
+ project.ant.fixcrlf(
+ file: target,
+ encoding: "UTF-8",
+ eol: "lf"
+ )
+ }
+ })
+
+ tasks.register("regenerateHTMLStripCharFilter", JFlexTask, {
+ description = "Regenerate HTMLStripCharFilter.java"
+
+ classpath = configurations.jflex
+
+ // HTMLStripCharFilter.jflex includes HTMLCharacterEntities.jflex so we make sure it's up to date.
+ dependsOn generateHTMLCharacterEntities
+
+ // Add included files as inputs.
+ inputs.file file('src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex')
+
+ jflexFile = file('src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex')
+ skeleton = skeletonDefault
+ })
+}
+
+//
+// Regenerate precomputed Packed* and bulkOperation* classes.
+//
+
+tasks.register("regeneratePacked64SingleBlock", {
+ description = "Regenerate Packed64SingleBlock and BulkOperation*.java"
+
+ def targetDir = file("src/java/org/apache/lucene/util/packed")
+
+ def packedResources = rootProject.file("gradle/regenerate/packed")
+ def templateFiles = [
+ file("${packedResources}/gen_BulkOperation.py"),
+ file("${packedResources}/gen_Packed64SingleBlock.py")
+ ]
+
+ inputs.files templateFiles
+ outputs.files fileTree(dir: targetDir, includes: [
+ "Packed64SingleBlock.java",
+ "BulkOperation*.java"
+ ])
+
+ doLast {
+ templateFiles.each {prog ->
+ logger.lifecycle("Executing: ${prog} in ${targetDir}")
+ buildGlobals.quietExec(it, {
+ workingDir = targetDir
+ executable = buildGlobals.externalTool("python3")
+ args = ['-B', "${prog}"]
+ })
+ }
+ // Correct line endings for Windows.
+ project.ant.fixcrlf(
+ srcDir: targetDir,
+ includes: 'Packed64SingleBlock.java, BulkOperation*.java',
+ encoding: 'UTF-8',
+ eol: 'lf'
+ )
+ }
+})
+
+//
+// Regenerate moman-based resources
+//
+
+def momanSource = "https://github.com/jpbarrette/moman/archive/497c90e34e412b6494db6dabf0d95db8034bd325.zip"
+def momanDir = file("${buildDir}/moman")
+
+def installMoman = tasks.register("installMoman", Download, {
+ def momanZip = file("${momanDir}/moman.zip")
+
+ src momanSource
+ dest momanZip
+ onlyIfModified true
+
+ doLast {
+ ant.unzip(src: momanZip, dest: momanDir, overwrite: "true") {
+ ant.cutdirsmapper(dirs: "1")
+ }
+ }
+})
+
+tasks.register("regenerateParametricDescription", {
+ dependsOn installMoman
+
+ description = "Regenerate *ParametricDescription"
+ regenerate {
+ ifSkippedAlsoSkip = ["installMoman"]
+ }
+
+ def momanResources = rootProject.file("gradle/regenerate/moman")
+ def targetDir = file("src/java/org/apache/lucene/util/automaton")
+ def templateFile = file("${momanResources}/createLevAutomata.py")
+
+ inputs.property("source", momanSource)
+ inputs.file templateFile
+ outputs.files fileTree(dir: targetDir, includes: ["*ParametricDescription.java"])
+
+ doLast {
+ ['1', '2'].each {num ->
+ ['True', 'False'].each {transpose ->
+ buildGlobals.quietExec(it, {
+ workingDir = targetDir
+ executable = buildGlobals.externalTool("python3")
+ args = [
+ '-B',
+ templateFile.toString(),
+ num,
+ transpose,
+ "${momanDir}/finenight/python"
+ ]
+ })
+ }
+ }
+ project.ant.fixcrlf(
+ srcDir: targetDir,
+ includes: '*ParametricDescription.java',
+ encoding: 'UTF-8',
+ eol: 'lf'
+ )
+ }
+})
diff --git a/lucene/core/src/generated/checksums/regeneratePacked64SingleBlock.json b/lucene/core/src/generated/checksums/regeneratePacked64SingleBlock.json
index 9942d9f77aa..0127428f6ed 100644
--- a/lucene/core/src/generated/checksums/regeneratePacked64SingleBlock.json
+++ b/lucene/core/src/generated/checksums/regeneratePacked64SingleBlock.json
@@ -1,6 +1,6 @@
{
- "gradle/regenerate/moman/gen_BulkOperation.py": "da9eccf0c8dd33325df5f1e76a1de74226035734",
- "gradle/regenerate/moman/gen_Packed64SingleBlock.py": "0fd2a498c3edbd195a68c2d1d686be0aed51a104",
+ "gradle/regenerate/packed/gen_BulkOperation.py": "da9eccf0c8dd33325df5f1e76a1de74226035734",
+ "gradle/regenerate/packed/gen_Packed64SingleBlock.py": "0fd2a498c3edbd195a68c2d1d686be0aed51a104",
"lucene/core/src/java/org/apache/lucene/util/packed/BulkOperation.java": "35a89e4f3dd37084cc059a784bcbeb384256773b",
"lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java": "2b0d9226bae8a07ce4970bcaa9d4d0cd4fe2c79a",
"lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked1.java": "c483aa35c275bacc1f3a010c5b879441be502108",
diff --git a/lucene/distribution.tests/build.gradle b/lucene/distribution.tests/build.gradle
index de4b2953361..76b73f7cb7a 100644
--- a/lucene/distribution.tests/build.gradle
+++ b/lucene/distribution.tests/build.gradle
@@ -37,7 +37,7 @@ dependencies {
moduleTestImplementation deps.assertj
}
-test {
+tasks.named("test").configure {
dependsOn configurations.binaryDistribution
// We need to pass the system property using a lazy provider, not supported at the moment:
diff --git a/lucene/expressions/build.gradle b/lucene/expressions/build.gradle
index 3fcfb7e4f84..b53cfdf7ff2 100644
--- a/lucene/expressions/build.gradle
+++ b/lucene/expressions/build.gradle
@@ -24,3 +24,91 @@ dependencies {
moduleImplementation deps.antlr.runtime
moduleTestImplementation project(':lucene:test-framework')
}
+
+//
+// Regenerate antlr4 grammars.
+//
+
+configurations {
+ antlr
+}
+
+dependencies {
+ antlr deps.antlr.core
+}
+
+tasks.register("regenerateAntlr", {
+ description = "Regenerate Javascript.g4"
+
+ dependsOn configurations.antlr
+
+ def generatedPatterns = [
+ "**/JavascriptBaseVisitor.java",
+ "**/JavascriptLexer.java",
+ "**/JavascriptParser.java",
+ "**/JavascriptVisitor.java",
+ ]
+
+ def sourceDir = "src/java/org/apache/lucene/expressions/js"
+ def inputFiles = file("$sourceDir/Javascript.g4")
+ def tempOutput = layout.buildDirectory.dir("antlr").get().asFile
+ def outputFiles = fileTree(dir: sourceDir, includes: generatedPatterns)
+
+ inputs.property "antlr-version", deps.antlr.core.get().version
+ inputs.files inputFiles
+ outputs.files outputFiles
+
+ doFirst {
+ buildGlobals.getFileOps().delete(tempOutput)
+ buildGlobals.getExecOps().javaexec {
+ mainClass = "org.antlr.v4.Tool"
+ classpath = configurations.antlr
+
+ ignoreExitValue = false
+ args = [
+ "-no-listener",
+ "-visitor",
+ "-package",
+ "org.apache.lucene.expressions.js",
+ "-o",
+ tempOutput,
+ inputFiles
+ ]
+ }
+
+ def generatedFiles = fileTree(dir: tempOutput, includes: generatedPatterns)
+ generatedFiles.each {file ->
+ buildGlobals.modifyFile(file, {text ->
+ text = text.replaceAll("public ((interface|class) Javascript\\w+)", "\$1")
+ text = text.replaceAll("// Generated from .*",
+ """
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// ANTLR GENERATED CODE: DO NOT EDIT.
+""")
+ return text
+ })
+ }
+
+ buildGlobals.getFileOps().copy({CopySpec spec ->
+ spec.from(tempOutput)
+ spec.into(sourceDir)
+ spec.include(generatedPatterns)
+ })
+ }
+})
diff --git a/lucene/queryparser/build.gradle b/lucene/queryparser/build.gradle
index 6e4fdb9443f..d3fed13119a 100644
--- a/lucene/queryparser/build.gradle
+++ b/lucene/queryparser/build.gradle
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-
+import org.apache.lucene.gradle.plugins.regenerate.JavaCCTask
description = 'Query parsers and parsing framework'
@@ -26,3 +26,216 @@ dependencies {
moduleTestImplementation project(':lucene:test-framework')
}
+
+//
+// This adds regeneration of javacc-generated sources
+//
+
+def commonCleanups = { FileTree generatedFiles ->
+ // This is a minor typo in a comment that nonetheless people have hand-corrected in the past.
+ generatedFiles.matching({include "CharStream.java"}).each {file ->
+ buildGlobals.modifyFile(file, {text ->
+ return text.replace(
+ "implemetation",
+ "implementation");
+ })
+ }
+
+ generatedFiles.each {file ->
+ buildGlobals.modifyFile(file, {text ->
+ // Normalize EOLs and tabs (EOLs are a side-effect of modifyFile).
+ text = text.replace("\t", " ");
+ text = text.replaceAll("JavaCC - OriginalChecksum=[^*]+", "(filtered)")
+ text = text.replace("StringBuffer", "StringBuilder")
+ return text
+ })
+ }
+
+ generatedFiles.matching({include "*TokenManager.java"}).each {file ->
+ buildGlobals.modifyFile(file, {text ->
+ // Remove redundant imports.
+ text = text.replaceAll(
+ /(?m)^import .+/,
+ "")
+ // Add CharStream imports.
+ text = text.replaceAll(
+ /package (.+)/,
+ '''
+ package $1
+ import org.apache.lucene.queryparser.charstream.CharStream;
+ '''.trim())
+ // Eliminates redundant cast message.
+ text = text.replace(
+ "int hiByte = (int)(curChar >> 8);",
+ "int hiByte = curChar >> 8;")
+ // Access to forbidden APIs.
+ text = text.replace(
+ "public java.io.PrintStream debugStream = System.out;",
+ "// (debugStream omitted).")
+ text = text.replace(
+ "public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }",
+ "// (setDebugStream omitted).")
+ // Remove dangling doc comments
+ text = text.replace(
+ "/** Debug output. */",
+ "// Debug output.")
+ text = text.replace(
+ "/** Set debug output. */",
+ "// Set debug output.")
+ return text
+ })
+ }
+}
+
+configurations.create("javacc")
+
+dependencies {
+ add("javacc", deps.javacc)
+}
+
+tasks.withType(JavaCCTask).configureEach {
+ classpath = configurations.javacc
+}
+
+tasks.register("regenerateClassicQueryParser", JavaCCTask, {
+ javaccFile = file('src/java/org/apache/lucene/queryparser/classic/QueryParser.jj')
+ description = "Regenerate classic query parser from ${javaccFile}"
+
+ afterGenerate(commonCleanups)
+ afterGenerate({ FileTree generatedFiles ->
+ generatedFiles.matching {include "QueryParser.java"}.each {file ->
+ buildGlobals.modifyFile(file, {text ->
+ text = text.replace(
+ "public QueryParser(CharStream ",
+ "protected QueryParser(CharStream ")
+ text = text.replace(
+ "public QueryParser(QueryParserTokenManager ",
+ "protected QueryParser(QueryParserTokenManager ")
+ text = text.replace(
+ "new java.util.ArrayList<int[]>",
+ "new java.util.ArrayList<>")
+ text = text.replace(
+ "final private LookaheadSuccess jj_ls =",
+ "static final private LookaheadSuccess jj_ls =")
+ text = text.replace(
+ "public class QueryParser ",
+ '@SuppressWarnings({"unused","null"}) public class QueryParser ')
+ text = text.replace(
+ "final public Query TopLevelQuery(",
+ "@Override final public Query TopLevelQuery(")
+ text = text.replace(
+ "public void ReInit(CharStream ",
+ "@Override public void ReInit(CharStream ")
+ return text
+ })
+ }
+ })
+})
+
+
+tasks.register("regenerateSurroundQueryParser", JavaCCTask, {
+ javaccFile = file('src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj')
+ description = "Regenerate surround query parser from ${javaccFile}"
+
+ classpath = configurations.javacc
+
+ afterGenerate(commonCleanups)
+
+ afterGenerate({FileTree generatedFiles ->
+ generatedFiles.matching {include "QueryParser.java"}.each {file ->
+ buildGlobals.modifyFile(file, {text ->
+ text = text.replace(
+ "import org.apache.lucene.analysis.TokenStream;",
+ "")
+ text = text.replace(
+ "new java.util.ArrayList<int[]>",
+ "new java.util.ArrayList<>")
+ text = text.replace(
+ "public class QueryParser ",
+ '@SuppressWarnings({"unused","null"}) public class QueryParser ')
+ return text
+ })
+ }
+ })
+})
+
+tasks.register("regenerateFlexibleStandardSyntaxParser", JavaCCTask, {
+ javaccFile = file('src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj')
+ description = "Regenerate flexible query parser from ${javaccFile}"
+
+ classpath = configurations.javacc
+
+ afterGenerate(commonCleanups)
+
+ afterGenerate({FileTree generatedFiles ->
+ generatedFiles.matching {include "ParseException.java"}.each {file ->
+ buildGlobals.modifyFile(file, {text ->
+ // Modify constructor.
+ text = text.replace(
+ "class ParseException extends Exception",
+ "class ParseException extends QueryNodeParseException")
+
+ // Modify imports.
+ text = text.replace(
+ "package org.apache.lucene.queryparser.flexible.standard.parser;", '''\
+ package org.apache.lucene.queryparser.flexible.standard.parser;
+
+ import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException;
+ import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages;
+ import org.apache.lucene.queryparser.flexible.messages.Message;
+ import org.apache.lucene.queryparser.flexible.messages.MessageImpl;
+ ''')
+
+ // Modify constructors and code bits
+ text = text.replaceAll(
+ /(?s)[ ]*public ParseException\(Token currentTokenVal[^}]+[}]/, '''\
+ public ParseException(Token currentTokenVal,
+ int[][] expectedTokenSequencesVal, String[] tokenImageVal)
+ {
+ super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, initialise(
+ currentTokenVal, expectedTokenSequencesVal, tokenImageVal)));
+ this.currentToken = currentTokenVal;
+ this.expectedTokenSequences = expectedTokenSequencesVal;
+ this.tokenImage = tokenImageVal;
+ }
+ ''')
+
+ text = text.replaceAll(
+ /(?s)[ ]*public ParseException\(String message\)[^}]+[}]/, '''\
+ public ParseException(Message message)
+ {
+ super(message);
+ }
+ ''')
+
+ text = text.replaceAll(
+ /(?s)[ ]*public ParseException\(\)[^}]+[}]/, '''\
+ public ParseException()
+ {
+ super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, "Error"));
+ }
+ ''')
+ return text
+ })
+ }
+
+ generatedFiles.matching {include "StandardSyntaxParser.java"}.each {file ->
+ buildGlobals.modifyFile(file, {text ->
+ // Remove redundant cast
+ text = text.replace(
+ "new java.util.ArrayList<int[]>",
+ "new java.util.ArrayList<>")
+ text = text.replace(
+ "new ArrayList<QueryNode>()",
+ "new ArrayList<>()")
+ text = text.replace(
+ "Collections.<QueryNode> singletonList",
+ "Collections.singletonList")
+ text = text.replace(
+ "public class StandardSyntaxParser ",
+ '@SuppressWarnings({"unused","null"}) public class StandardSyntaxParser ')
+ return text
+ })
+ }
+ })
+})
diff --git a/lucene/test-framework/build.gradle b/lucene/test-framework/build.gradle
index 54128c52427..36687e87b83 100644
--- a/lucene/test-framework/build.gradle
+++ b/lucene/test-framework/build.gradle
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-
+import org.apache.lucene.gradle.plugins.misc.QuietExec
description = 'Framework for testing Lucene-based applications'
@@ -32,3 +32,51 @@ dependencies {
moduleImplementation project(':lucene:codecs')
}
+
+//
+// Regenerate unicode resources
+//
+
+tasks.register("regenerateEmojiTokenizationTest", QuietExec, {
+ def unicodeVersion = "12.1"
+
+ def genDir = file("src/java/org/apache/lucene/tests/analysis/standard")
+ def genScript = file("${genDir}/generateEmojiTokenizationTest.pl")
+ def genOutput = file("${genDir}/EmojiTokenizationTestUnicode_${unicodeVersion.replace('.', '_')}.java")
+
+ description = "Regenerate ${genOutput}"
+
+ inputs.file genScript
+ inputs.property "unicodeVersion", unicodeVersion
+ outputs.file genOutput
+
+ workingDir = genDir
+ executable = buildGlobals.externalTool("perl")
+ args = [
+ genScript,
+ "-v",
+ unicodeVersion
+ ]
+})
+
+tasks.register("regenerateJavaUnicodeWordBreakTest", QuietExec, {
+ def unicodeVersion = "12.1"
+
+ def genDir = file("src/java/org/apache/lucene/tests/analysis/standard")
+ def genScript = file("${genDir}/generateJavaUnicodeWordBreakTest.pl")
+ def genOutput = file("${genDir}/WordBreakTestUnicode_${unicodeVersion.replace('.', '_')}.java")
+
+ description = "Regenerate ${genOutput}"
+
+ inputs.file genScript
+ inputs.property "unicodeVersion", unicodeVersion
+ outputs.file genOutput
+
+ workingDir = genDir
+ executable = buildGlobals.externalTool("perl")
+ args = [
+ genScript,
+ "-v",
+ unicodeVersion
+ ]
+})