You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2025/09/27 09:15:16 UTC
(lucene) branch main updated: Port and modernize gradle's changes-to-html (#15235)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 92888a75b44 Port and modernize gradle's changes-to-html (#15235)
92888a75b44 is described below
commit 92888a75b44caf71255e6d2e5b69cf7bce8954c9
Author: Dawid Weiss <da...@carrotsearch.com>
AuthorDate: Sat Sep 27 11:15:11 2025 +0200
Port and modernize gradle's changes-to-html (#15235)
---
build-tools/build-infra-shadow/build.gradle | 5 +
.../lucene.datasets.external-datasets.gradle | 197 ---------------------
.../lucene.documentation.changes-to-html.gradle | 128 -------------
.../apache/lucene/gradle/datasets/ExtractZstd.java | 20 ++-
.../plugins/documentation/ChangesToHtmlTask.java | 146 +++++++++++++++
build.gradle | 3 -
.../documentation/changes-to-html/changes2html.pl | 2 +-
lucene/benchmark/build.gradle | 130 +++++++++++++-
lucene/documentation/build.gradle | 33 ++++
9 files changed, 333 insertions(+), 331 deletions(-)
diff --git a/build-tools/build-infra-shadow/build.gradle b/build-tools/build-infra-shadow/build.gradle
index c6291c69624..cdbad83d927 100644
--- a/build-tools/build-infra-shadow/build.gradle
+++ b/build-tools/build-infra-shadow/build.gradle
@@ -42,6 +42,11 @@ static Provider<String> plugin(Provider<PluginDependency> plugin) {
dependencies {
implementation gradleApi()
implementation localGroovy()
+
+ // seems to be required by gradle apis, otherwise compilation fails even though
+ // we don't use it.
+ compileOnly ("com.google.code.findbugs:findbugs-annotations:3.0.1")
+
implementation deps.commons.codec
implementation deps.randomizedtesting.runner
implementation deps.zstd
diff --git a/build-tools/build-infra/src/main/groovy/lucene.datasets.external-datasets.gradle b/build-tools/build-infra/src/main/groovy/lucene.datasets.external-datasets.gradle
deleted file mode 100644
index 981d4cffba7..00000000000
--- a/build-tools/build-infra/src/main/groovy/lucene.datasets.external-datasets.gradle
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.nio.file.Files
-import com.github.luben.zstd.ZstdInputStream
-import de.undercouch.gradle.tasks.download.Download
-import org.apache.lucene.gradle.datasets.ExtractReuters
-
-// tasks that download various benchmark/test data sets from the internet.
-
-// TODO: not sure whether this should live in benchmarks, but for now let it be.
-
-if (project != project.rootProject) {
- throw new GradleException("Applicable to rootProject only: " + project.path)
-}
-
-def static unzstd(java.nio.file.Path src, java.nio.file.Path dst) {
- new ZstdInputStream(new BufferedInputStream(Files.newInputStream(src))).withCloseable { is ->
- new BufferedOutputStream(Files.newOutputStream(dst)).withCloseable { os ->
- is.transferTo(os)
- }
- }
-}
-
-configure(project(":lucene:benchmark")) {
- def dataDir = project.file("work")
-
- def getEnWikiTask = tasks.register("getEnWiki", Download, {
- ext {
- name = "enwiki-20070527-pages-articles.xml"
- src = "https://s3.amazonaws.com/lucene-testdata/wikipedia/${name}.zst"
- intermediate = file("${dataDir}/${name}.bz2")
- dst = file("${dataDir}/${name}")
- }
-
- description = "Download the ${ext.name} data set."
-
- outputs.file ext.dst
-
- src ext.src
- dest ext.intermediate
- overwrite false
- compress false
-
- doLast {
- logger.lifecycle("Decompressing ${ext.name}...")
- unzstd(ext.intermediate.toPath(), ext.dst.toPath())
- }
- })
-
- def getEnWikiRandomLinesTask = tasks.register("getEnWikiRandomLines", Download, {
- ext {
- name = "enwiki.random.lines.txt"
- src = "https://s3.amazonaws.com/lucene-testdata/wikipedia/${name}.zst"
- intermediate = file("${dataDir}/${name}.zst")
- dst = file("${dataDir}/${name}")
- }
-
- description = "Download the ${ext.name} data set."
-
- outputs.file ext.dst
-
- src ext.src
- dest ext.intermediate
- overwrite false
- compress false
-
- doLast {
- logger.lifecycle("Decompressing ${ext.name}...")
- unzstd(ext.intermediate.toPath(), ext.dst.toPath())
- }
- })
-
- def getGeoNamesTask = tasks.register("getGeoNames", Download, {
- // note: latest data is at: https://download.geonames.org/export/dump/allCountries.zip
- // and then randomize with: gsort -R -S 1500M file.txt > file_random.txt
- // and then compress with: bzip2 -9 -k file_random.txt
- ext {
- name = "geonames_20130921_randomOrder_allCountries.txt"
- src = "https://s3.amazonaws.com/lucene-testdata/geonames/${name}.bz2"
- intermediate = file("${dataDir}/${name}.bz2")
- dst = file("${dataDir}/${name}")
- }
-
- description = "Download the ${ext.name} data set."
-
- outputs.file ext.dst
-
- src ext.src
- dest ext.intermediate
- overwrite false
- compress false
-
- doLast {
- logger.lifecycle("Decompressing ${ext.name}...")
- ant.bunzip2(src: ext.intermediate, dest: ext.dst)
- }
- })
-
- def getTop100kWikiWordFilesTask = tasks.register("getTop100kWikiWordFiles", Download, {
- ext {
- name = "top.100k.words.de.en.fr.uk.wikipedia.2009-11"
- src = "https://s3.amazonaws.com/lucene-testdata/wikipedia/${name}.tar.bz2"
- intermediate = file("${dataDir}/${name}.bz2")
- dst = file("${dataDir}/${name}")
- }
-
- description = "Download the ${ext.name} data set."
-
- outputs.dir ext.dst
-
- src ext.src
- dest ext.intermediate
- overwrite false
- compress false
-
- doLast {
- logger.lifecycle("Decompressing ${ext.name}...")
- project.sync {
- from tarTree(ext.intermediate) // defined above. Will decompress on the fly
- into ext.dst
- }
- }
- })
-
- def getReutersTask = tasks.register("getReuters", Download, {
- ext {
- name = "reuters21578"
- src = "https://kdd.ics.uci.edu/databases/${name}/${name}.tar.gz"
- intermediate = file("${dataDir}/${name}.tar.gz")
- dst = file("${dataDir}/reuters-out")
- }
-
- description = "Download the ${ext.name} data set."
-
- outputs.dir ext.dst
-
- src ext.src
- dest ext.intermediate
- overwrite false
- compress false
-
- doLast {
- def untarPath = file("$temporaryDir/reuters-untar")
-
- logger.lifecycle("Decompressing ${ext.name}...")
- project.sync {
- from(tarTree(intermediate)) {
- exclude '*.txt'
- }
- into untarPath
- }
-
- logger.lifecycle("Extracting ${ext.name} into ${ext.dst}...")
- ext.dst.deleteDir()
- ExtractReuters.main(untarPath.toString(), ext.dst.toString())
- }
- })
-
- def downloadTasks = [
- getEnWikiTask,
- getGeoNamesTask,
- getTop100kWikiWordFilesTask,
- getReutersTask,
- getEnWikiRandomLinesTask
- ]
-
- tasks.register("downloadDatasets", {
- group = "Data set download"
- description = "Download all data sets."
-
- dependsOn downloadTasks
- })
-
- downloadTasks.each {
- it.configure {
- group = "Data set download"
- doFirst {
- logger.lifecycle("Downloading data set ${it.ext.name} from ${it.ext.src} to ${it.ext.dst}...")
- }
- }
- }
-}
diff --git a/build-tools/build-infra/src/main/groovy/lucene.documentation.changes-to-html.gradle b/build-tools/build-infra/src/main/groovy/lucene.documentation.changes-to-html.gradle
deleted file mode 100644
index 98462b5436a..00000000000
--- a/build-tools/build-infra/src/main/groovy/lucene.documentation.changes-to-html.gradle
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-if (project != project.rootProject) {
- throw new GradleException("Applicable to rootProject only: " + project.path)
-}
-
-File resources = rootProject.file("gradle/documentation/changes-to-html")
-
-configure(project(':lucene:documentation')) {
- def changesToHtmlTask = tasks.register("changesToHtml", ChangesToHtmlTask, {
- siteDir = resources
- script = file("${resources}/changes2html.pl")
- })
-
- // Make the rendered HTML of changes available as a separate
- // artifact for the distribution.
- configurations {
- changesHtml
- }
-
- artifacts {
- changesHtml changesToHtmlTask.get().targetDir, {
- builtBy changesToHtmlTask
- }
- }
-}
-
-// compile changes.txt into an html file
-class ChangesToHtmlTask extends DefaultTask {
-
- @Internal
- Project productProject = project.parent
-
- @Internal
- String productName = productProject.name
-
- @InputFile
- File changesFile = productProject.file('CHANGES.txt')
-
- @InputFile
- File changesDoapFile = project.rootProject.file("dev-tools/doap/${productName}.rdf")
-
- @InputDirectory
- File siteDir
-
- @OutputDirectory
- final DirectoryProperty targetDir = project.objects.directoryProperty().fileProvider(
- project.providers.provider { project.file("${project.ext.docroot}/changes") })
-
- @Input
- def luceneDocUrl = "${-> project.luceneDocUrl }"
-
- @InputFile
- def script
-
- private ExecOperations execOperations;
-
- @Inject
- ChangesToHtmlTask(ExecOperations execOperations) {
- this.execOperations = execOperations
- }
-
- def loadVersions(File outfile) {
- // load version properties from DOAP RDF
- def prefix = "doap.${productName}".toString()
- ant.xmlproperty(keeproot: false, file: changesDoapFile, collapseAttributes: false, prefix: "${prefix}")
- outfile.withWriter("UTF-8") { writer ->
- writer.println(ant.properties["${prefix}.Project.release.Version.revision"])
- writer.println(ant.properties["${prefix}.Project.release.Version.created"])
- }
- }
-
- def toHtml(File versionsFile) {
- def output = new ByteArrayOutputStream()
- def result = execOperations.exec {
- executable = project.buildGlobals.externalTool("perl")
- standardInput = changesFile.newInputStream()
- standardOutput = project.file("${targetDir.get().getAsFile()}/Changes.html").newOutputStream()
- errorOutput = output
- ignoreExitValue = true
-
- args += [
- "-CSD",
- script,
- "${productName}",
- versionsFile.toString(),
- luceneDocUrl.concat('/') // slash required at end by perl script
- ]
- }
-
- if (result.getExitValue() != 0) {
- throw new GradleException("Changes generation failed:\n${output}")
- }
- }
-
- @TaskAction
- def convert() {
- project.mkdir targetDir
- if (changesFile.exists() && changesDoapFile.exists()) {
- File versionsFile = project.file("${project.buildDir}/doap.${project.name}.changes.version.dates.csv")
- loadVersions(versionsFile)
- toHtml(versionsFile)
- project.copy {
- from siteDir
- into targetDir
- include "*.css"
- }
- versionsFile.delete()
- } else {
- throw new GradleException("Changes file ${changesFile} or Doap file ${changesDoapFile} not found.")
- }
- }
-}
diff --git a/lucene/documentation/build.gradle b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/datasets/ExtractZstd.java
similarity index 56%
copy from lucene/documentation/build.gradle
copy to build-tools/build-infra/src/main/java/org/apache/lucene/gradle/datasets/ExtractZstd.java
index 4eb6bf60ade..cf06e86828c 100644
--- a/lucene/documentation/build.gradle
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/datasets/ExtractZstd.java
@@ -14,5 +14,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.lucene.gradle.datasets;
-description = 'Lucene HTML documentation'
+import com.github.luben.zstd.ZstdInputStream;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/** Utility for zstd decompression */
+public final class ExtractZstd {
+ private ExtractZstd() {}
+
+ public static void unzstd(Path src, Path dst) throws IOException {
+ try (var is = new ZstdInputStream(new BufferedInputStream(Files.newInputStream(src)));
+ var os = new BufferedOutputStream(Files.newOutputStream(dst))) {
+ is.transferTo(os);
+ }
+ }
+}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/documentation/ChangesToHtmlTask.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/documentation/ChangesToHtmlTask.java
new file mode 100644
index 00000000000..2be8ff417a5
--- /dev/null
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/documentation/ChangesToHtmlTask.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.gradle.plugins.documentation;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import javax.inject.Inject;
+import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension;
+import org.apache.tools.ant.Project;
+import org.apache.tools.ant.taskdefs.XmlProperty;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.GradleException;
+import org.gradle.api.file.DirectoryProperty;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.internal.file.FileOperations;
+import org.gradle.api.provider.Property;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.InputDirectory;
+import org.gradle.api.tasks.InputFile;
+import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.process.ExecOperations;
+import org.gradle.process.ExecResult;
+
+public abstract class ChangesToHtmlTask extends DefaultTask {
+ @Input
+ public abstract Property<String> getProductName();
+
+ @InputFile
+ public abstract RegularFileProperty getChangesFile();
+
+ @Inject
+ public abstract ExecOperations getExecOps();
+
+ @InputFile
+ public abstract RegularFileProperty getChangesDoapFile();
+
+ @InputDirectory
+ public abstract DirectoryProperty getSiteDir();
+
+ @OutputDirectory
+ public abstract DirectoryProperty getTargetDir();
+
+ @InputFile
+ public abstract RegularFileProperty getConversionScript();
+
+ @Inject
+ public abstract FileOperations getFileOperations();
+
+ @Inject
+ public ChangesToHtmlTask() {}
+
+ @TaskAction
+ public void convert() throws IOException {
+ Path doapFile = getChangesDoapFile().get().getAsFile().toPath();
+
+ Path versionsFile = loadVersions(doapFile);
+ toHtml(versionsFile);
+
+ getFileOperations()
+ .copy(
+ spec -> {
+ spec.from(getSiteDir());
+ spec.into(getTargetDir());
+ spec.include("*.css");
+ });
+ }
+
+ private void toHtml(Path versionsFile) throws IOException {
+ try (var baos = new ByteArrayOutputStream();
+ var changesFileIs = Files.newInputStream(getChangesFile().get().getAsFile().toPath());
+ var output =
+ Files.newOutputStream(getTargetDir().get().file("Changes.html").getAsFile().toPath())) {
+ ExecResult result =
+ getExecOps()
+ .exec(
+ spec -> {
+ spec.setExecutable(
+ getProject()
+ .getExtensions()
+ .getByType(LuceneBuildGlobalsExtension.class)
+ .externalTool("perl"));
+ spec.setStandardInput(changesFileIs);
+ spec.setStandardOutput(output);
+ spec.setErrorOutput(output);
+ spec.setIgnoreExitValue(true);
+
+ spec.setArgs(
+ List.of(
+ "-CSD",
+ getConversionScript().get().getAsFile().toString(),
+ getProductName().get(),
+ versionsFile.toString()));
+ });
+
+ if (result.getExitValue() != 0) {
+ throw new GradleException(
+ "Changes generation failed:\n" + baos.toString(StandardCharsets.UTF_8));
+ }
+ }
+ }
+
+ /** load version properties from DOAP RDF */
+ private Path loadVersions(Path changesDoapFile) throws IOException {
+ if (!Files.isRegularFile(changesDoapFile)) {
+ throw new GradleException("doapFile does not exist: " + changesDoapFile);
+ }
+
+ Path tempFile = getTemporaryDirFactory().create().toPath().resolve("versions.properties");
+ try (var writer = Files.newBufferedWriter(tempFile)) {
+ String prefix = "doap." + getProductName().get();
+
+ Project antProject = getAnt().getAntProject();
+ XmlProperty xmlPropertyTask = (XmlProperty) antProject.createTask("xmlproperty");
+ xmlPropertyTask.setKeeproot(false);
+ xmlPropertyTask.setFile(changesDoapFile.toFile());
+ xmlPropertyTask.setCollapseAttributes(false);
+ xmlPropertyTask.setPrefix(prefix);
+ xmlPropertyTask.execute();
+
+ writer.write(antProject.getProperty(prefix + ".Project.release.Version.revision"));
+ writer.newLine();
+ writer.write(antProject.getProperty(prefix + ".Project.release.Version.created"));
+ writer.newLine();
+ }
+ return tempFile;
+ }
+}
diff --git a/build.gradle b/build.gradle
index b483f83f1db..dddc62b3325 100644
--- a/build.gradle
+++ b/build.gradle
@@ -39,10 +39,7 @@ plugins {
id "lucene.misc.pylucene"
- id "lucene.datasets.external-datasets"
-
id "lucene.documentation"
- id "lucene.documentation.changes-to-html"
id "lucene.documentation.markdown"
id "lucene.documentation.render-javadoc"
diff --git a/gradle/documentation/changes-to-html/changes2html.pl b/gradle/documentation/changes-to-html/changes2html.pl
index 5814c252ac3..6610d358d71 100644
--- a/gradle/documentation/changes-to-html/changes2html.pl
+++ b/gradle/documentation/changes-to-html/changes2html.pl
@@ -50,7 +50,7 @@ if ($product !~ /^(LUCENE|SOLR)$/) {
exit(1);
}
my %release_dates = &setup_release_dates($ARGV[1]);
-my $lucene_javadoc_url = ($product eq 'SOLR' ? $ARGV[2] : ''); # Only Solr supplies this on the cmdline
+my $lucene_javadoc_url = ''; # Only Solr used to supply this on the cmdline
my $in_major_component_versions_section = 0;
diff --git a/lucene/benchmark/build.gradle b/lucene/benchmark/build.gradle
index 4e98fc344d8..ffc443e39e0 100644
--- a/lucene/benchmark/build.gradle
+++ b/lucene/benchmark/build.gradle
@@ -15,6 +15,10 @@
* limitations under the License.
*/
+import de.undercouch.gradle.tasks.download.Download
+import org.apache.lucene.gradle.datasets.ExtractReuters
+import org.apache.lucene.gradle.datasets.ExtractZstd
+
plugins {
id "java"
}
@@ -42,7 +46,7 @@ dependencies {
})
// LUCENE-10337: Include xercesImpl on regular classpath where it won't cause conflicts.
- implementation (deps.xerces, {
+ implementation(deps.xerces, {
exclude module: "xml-apis"
})
@@ -90,3 +94,127 @@ tasks.register("run", JavaExec, {
suspend = true
}
})
+
+def dataDir = project.layout.projectDirectory.dir("work")
+
+/**
+ * Registers a download-and-unpack dataset task with a given strategy.
+ */
+def registerDataset = { String taskName, Map<String, Object> cfg ->
+ return tasks.register(taskName, Download, {
+ String name = cfg.name
+ String src = cfg.src
+ File dst = cfg.dst
+
+ def archiveName = src.substring(src.lastIndexOf('/') + 1)
+ File intermediate = dataDir.file(archiveName).asFile
+
+ group = "Data set download"
+ description = "Download the ${name} data set."
+
+ if (cfg.outType == "dir") {
+ outputs.dir(dst)
+ } else {
+ outputs.file(dst)
+ }
+
+ it.src(src)
+ it.dest(intermediate)
+ it.overwrite(false)
+ it.compress(false)
+
+ doFirst {
+ logger.lifecycle("Downloading data set $name} from ${src} to ${dst}...")
+ }
+
+ doLast {
+ logger.lifecycle("Decompressing ${name}...")
+ switch (cfg.strategy) {
+ case 'zstd':
+ ExtractZstd.unzstd(intermediate.toPath(), dst.toPath())
+ break
+
+ case 'bunzip2':
+ ant.bunzip2(src: intermediate, dest: dst)
+ break
+
+ case 'tar':
+ project.sync {
+ from tarTree(intermediate)
+ into dst
+ }
+ break
+
+ case 'reuters':
+ File untarPath = file("${temporaryDir}/reuters-untar")
+ project.sync {
+ from(tarTree(intermediate)) {
+ exclude '*.txt'
+ }
+ into untarPath
+ }
+ logger.lifecycle("Extracting ${name} into ${dst}...")
+ dst.deleteDir()
+ ExtractReuters.main(untarPath.toString(), dst.toString())
+ break
+
+ default:
+ throw new GradleException("Unknown strategy: ${cfg.strategy}")
+ }
+ }
+ })
+}
+
+def datasets = [
+ getEnWiki: [
+ name: "enwiki-20070527-pages-articles.xml",
+ src: "https://s3.amazonaws.com/lucene-testdata/wikipedia/enwiki-20070527-pages-articles.xml.zst",
+ dst: dataDir.file("enwiki-20070527-pages-articles.xml").asFile,
+ outType: "file",
+ strategy: "zstd"
+ ],
+
+ getEnWikiRandomLines: [
+ name: "enwiki.random.lines.txt",
+ src: "https://s3.amazonaws.com/lucene-testdata/wikipedia/enwiki.random.lines.txt.zst",
+ dst: file("${dataDir}/enwiki.random.lines.txt"),
+ outType: "file",
+ strategy: "zstd"
+ ],
+
+ getGeoNames: [
+ name: "geonames_20130921_randomOrder_allCountries.txt",
+ src: "https://s3.amazonaws.com/lucene-testdata/geonames/geonames_20130921_randomOrder_allCountries.txt.bz2",
+ dst: file("${dataDir}/geonames_20130921_randomOrder_allCountries.txt"),
+ outType: "file",
+ strategy: "bunzip2"
+ ],
+
+ getTop100kWikiWordFiles: [
+ name: "top.100k.words.de.en.fr.uk.wikipedia.2009-11",
+ src: "https://s3.amazonaws.com/lucene-testdata/wikipedia/top.100k.words.de.en.fr.uk.wikipedia.2009-11.tar.bz2",
+ dst: file("${dataDir}/top.100k.words.de.en.fr.uk.wikipedia.2009-11"),
+ outType: "dir",
+ strategy: "tar"
+ ],
+
+ getReuters: [
+ name: "reuters21578",
+ src: "https://kdd.ics.uci.edu/databases/reuters21578/reuters21578.tar.gz",
+ dst: file("${dataDir}/reuters-out"),
+ outType: "dir",
+ strategy: "reuters"
+ ],
+]
+
+// Register all dataset downloading tasks
+def downloadTasks = datasets.collectEntries {taskName, cfg ->
+ [(taskName): registerDataset(taskName as String, cfg)]
+}.values()
+
+// An aggregate task to fetch them all.
+tasks.register("downloadDatasets") {
+ group = "Data set download"
+ description = "Download all data sets."
+ dependsOn downloadTasks
+}
diff --git a/lucene/documentation/build.gradle b/lucene/documentation/build.gradle
index 4eb6bf60ade..3442ce2b474 100644
--- a/lucene/documentation/build.gradle
+++ b/lucene/documentation/build.gradle
@@ -1,3 +1,5 @@
+import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -15,4 +17,35 @@
* limitations under the License.
*/
+import org.apache.lucene.gradle.plugins.documentation.ChangesToHtmlTask
+
description = 'Lucene HTML documentation'
+
+var docroot = layout.buildDirectory.dir("site")
+
+var changesToHtmlTask = tasks.register("changesToHtml", ChangesToHtmlTask, {
+ var productProject = getProject().project(":lucene")
+ var name = productProject.getName()
+
+ productName = name
+ changesFile = productProject.getLayout().getProjectDirectory().file("CHANGES.txt")
+ changesDoapFile = rootProject.layout.projectDirectory.file("dev-tools/doap/${name}.rdf")
+
+ siteDir = rootProject.layout.projectDirectory.dir("gradle/documentation/changes-to-html")
+ targetDir = docroot.map {dir -> dir.dir("changes")}
+
+ conversionScript = siteDir.file("changes2html.pl")
+})
+
+// Make the rendered HTML of changes available as a separate
+// artifact for the distribution.
+configurations {
+ changesHtml
+}
+
+artifacts {
+ changesHtml changesToHtmlTask.get().targetDir, {
+ builtBy changesToHtmlTask
+ }
+}
+