You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by th...@apache.org on 2020/09/14 16:38:20 UTC
[lucene-solr] 02/39: LUCENE-9312: Allow builds against arbitrary
JVMs (squashed jira/LUCENE-9312)
This is an automated email from the ASF dual-hosted git repository.
thelabdude pushed a commit to branch reference_impl_gradle_updates
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 4a47c3c80885ae9ed01d061a711c31a23624160f
Author: Dawid Weiss <dw...@apache.org>
AuthorDate: Tue Jul 21 09:19:38 2020 +0200
LUCENE-9312: Allow builds against arbitrary JVMs (squashed
jira/LUCENE-9312)
---
build.gradle | 19 +--
gradle/{ => documentation}/render-javadoc.gradle | 21 ++-
gradle/help.gradle | 1 +
gradle/testing/alternative-jdk-support.gradle | 72 ++++++++++
gradle/testing/runtime-jvm-support.gradle | 52 -------
help/jvms.txt | 18 +++
help/tests.txt | 15 --
lucene/CHANGES.txt | 174 ++++++++++++++++++++++-
8 files changed, 285 insertions(+), 87 deletions(-)
diff --git a/build.gradle b/build.gradle
index 9c07bc7..288a6a7 100644
--- a/build.gradle
+++ b/build.gradle
@@ -94,7 +94,7 @@ apply from: file('gradle/defaults-java.gradle')
apply from: file('gradle/testing/defaults-tests.gradle')
apply from: file('gradle/testing/randomization.gradle')
apply from: file('gradle/testing/fail-on-no-tests.gradle')
-apply from: file('gradle/testing/runtime-jvm-support.gradle')
+apply from: file('gradle/testing/alternative-jdk-support.gradle')
apply from: file('gradle/jar-manifest.gradle')
// Maven publishing.
@@ -150,21 +150,6 @@ apply from: file('gradle/documentation/documentation.gradle')
apply from: file('gradle/documentation/changes-to-html.gradle')
apply from: file('gradle/documentation/markdown.gradle')
apply from: file('gradle/render-javadoc.gradle')
+apply from: file('gradle/documentation/render-javadoc.gradle')
apply from: file('gradle/hacks/findbugs.gradle')
-allprojects {
- task ufclasspath {
- doLast{
- File ufPath = new File(project.getRootDir().getParentFile(), "unitflier/run/solr");
- if (configurations.hasProperty('testRuntimeClasspath')) {
- java.io.File file = new java.io.File(ufPath, project.projectDir.name + '.txt');
- file.getParentFile().mkdirs();
- file.write project.projectDir.toString() + "\n"
- file << sourceSets.test.output.classesDirs.asPath + "\n"
- file << project.projectDir.toString() + "/src/test-files" + ":" + project.projectDir.toString() + "/src/resources" + ":" + sourceSets.main.output.classesDirs.asPath + ":"
- file << sourceSets.test.output.classesDirs.asPath + ":"
- file << configurations.testRuntimeClasspath.asPath + "\n"
- }
- }
- }
-}
diff --git a/gradle/render-javadoc.gradle b/gradle/documentation/render-javadoc.gradle
similarity index 95%
rename from gradle/render-javadoc.gradle
rename to gradle/documentation/render-javadoc.gradle
index 7fcee58..27d282b 100644
--- a/gradle/render-javadoc.gradle
+++ b/gradle/documentation/render-javadoc.gradle
@@ -1,3 +1,5 @@
+import javax.annotation.Nullable
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -150,6 +152,11 @@ class RenderJavadocTask extends DefaultTask {
@Input
def solrDocUrl = "${->project.solrDocUrl}"
+
+ @Nullable
+ @Optional
+ @Input
+ def executable
/** Utility method to recursively collect all tasks with same name like this one that we depend on */
private Set findRenderTasksInDependencies() {
@@ -165,8 +172,6 @@ class RenderJavadocTask extends DefaultTask {
@TaskAction
public void render() {
- def javadocCmd = org.gradle.internal.jvm.Jvm.current().getJavadocExecutable()
-
def srcDirs = srcDirSet.srcDirs.findAll { dir -> dir.exists() }
def optionsFile = project.file("${getTemporaryDir()}/javadoc-options.txt")
@@ -254,6 +259,18 @@ class RenderJavadocTask extends DefaultTask {
}
})
+ def javadocCmd = {
+ if (executable == null) {
+ JavaInstallationRegistry registry = project.extensions.getByType(JavaInstallationRegistry)
+ JavaInstallation currentJvm = registry.installationForCurrentVirtualMachine.get()
+ return currentJvm.jdk.get().javadocExecutable.asFile
+ } else {
+ return project.file(executable)
+ }
+ }()
+
+ logger.info("Javadoc executable used: ${javadocCmd}")
+
def outputFile = project.file("${getTemporaryDir()}/javadoc-output.txt")
def result
outputFile.withOutputStream { output ->
diff --git a/gradle/help.gradle b/gradle/help.gradle
index d76ce90..edee1c3 100644
--- a/gradle/help.gradle
+++ b/gradle/help.gradle
@@ -22,6 +22,7 @@ configure(rootProject) {
["Workflow", "help/workflow.txt", "Typical workflow commands."],
["Ant", "help/ant.txt", "Ant-gradle migration help."],
["Tests", "help/tests.txt", "Tests, filtering, beasting, etc."],
+ ["Jvms", "help/jvms.txt", "Using alternative or EA JVM toolchains."],
["Deps", "help/dependencies.txt", "Declaring, inspecting and excluding dependencies."],
["ForbiddenApis", "help/forbiddenApis.txt", "How to add/apply rules for forbidden APIs."],
["LocalSettings", "help/localSettings.txt", "Local settings, overrides and build performance tweaks."],
diff --git a/gradle/testing/alternative-jdk-support.gradle b/gradle/testing/alternative-jdk-support.gradle
new file mode 100644
index 0000000..1e69291
--- /dev/null
+++ b/gradle/testing/alternative-jdk-support.gradle
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This adds support for compiling and testing against a different Java runtime.
+// This is the only way to build against JVMs not yet supported by Gradle itself.
+
+JavaInstallationRegistry registry = extensions.getByType(JavaInstallationRegistry)
+
+JavaInstallation currentJvm = registry.installationForCurrentVirtualMachine.get()
+
+JavaInstallation altJvm = {
+ def runtimeJavaHome = propertyOrDefault("runtime.java.home", System.getenv('RUNTIME_JAVA_HOME'))
+ if (!runtimeJavaHome) {
+ return currentJvm
+ } else {
+ return registry.installationForDirectory(
+ layout.projectDirectory.dir(runtimeJavaHome)).get()
+ }
+}()
+
+if (!currentJvm.javaExecutable.equals(altJvm.javaExecutable)) {
+ // Set up java toolchain tasks to use the alternative Java.
+ // This is a related Gradle issue for the future:
+ // https://github.com/gradle/gradle/issues/1652
+
+ configure(rootProject) {
+ task altJvmWarning() {
+ doFirst {
+ logger.warn("""NOTE: Alternative java toolchain will be used for compilation and tests:
+ Project will use Java ${altJvm.javaVersion} from: ${altJvm.installationDirectory}
+ Gradle runs with Java ${currentJvm.javaVersion} from: ${currentJvm.installationDirectory}
+""")
+ }
+ }
+ }
+
+ // Set up toolchain-dependent tasks to use the alternative JVM.
+ allprojects {
+ // Any tests
+ tasks.withType(Test) {
+ dependsOn ":altJvmWarning"
+ executable = altJvm.javaExecutable
+ }
+
+ // Any javac compilation tasks
+ tasks.withType(JavaCompile) {
+ dependsOn ":altJvmWarning"
+ options.fork = true
+ options.forkOptions.javaHome = altJvm.installationDirectory.asFile
+ }
+
+ def javadocExecutable = altJvm.jdk.get().javadocExecutable.asFile
+ tasks.matching { it.name == "renderJavadoc" || it.name == "renderSiteJavadoc" }.all {
+ dependsOn ":altJvmWarning"
+ executable = javadocExecutable
+ }
+ }
+}
diff --git a/gradle/testing/runtime-jvm-support.gradle b/gradle/testing/runtime-jvm-support.gradle
deleted file mode 100644
index ce48b56..0000000
--- a/gradle/testing/runtime-jvm-support.gradle
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This adds support for compiling and testing against a different Java runtime.
-// This is the only way to build against JVMs not yet supported by Gradle itself.
-
-import org.gradle.internal.jvm.Jvm
-
-def jvmForTests = {
- def runtimeJavaHome = propertyOrDefault("runtime.java.home", System.getenv('RUNTIME_JAVA_HOME'))
- if (!runtimeJavaHome) {
- return Jvm.current()
- } else {
- return Jvm.forHome(file(runtimeJavaHome))
- }
-}()
-def jvmGradle = Jvm.current()
-
-def differentTestJvm = (jvmGradle.javaHome.canonicalPath != jvmForTests.javaHome.canonicalPath)
-
-// Set up tasks to use the alternative Java.
-if (differentTestJvm) {
- configure(rootProject) {
- task testJvmWarning() {
- doFirst {
- logger.warn("This Java will be used for running tests: ${jvmForTests.javaExecutable}")
- }
- }
- }
-
- // Set up test tasks to use the alternative JVM.
- allprojects {
- tasks.withType(Test) {
- dependsOn ":testJvmWarning"
- executable = jvmForTests.javaExecutable
- }
- }
-}
\ No newline at end of file
diff --git a/help/jvms.txt b/help/jvms.txt
new file mode 100644
index 0000000..42a88d2
--- /dev/null
+++ b/help/jvms.txt
@@ -0,0 +1,18 @@
+Compiling and testing against different JVMs
+============================================
+
+By default tests are executed with the same Java Gradle is using internally.
+
+To run tests against a different Java version, define a property called
+"runtime.java.home" or define an environment variable "RUNTIME_JAVA_HOME"
+pointing at the JDK installation folder.
+
+If property is being used, it can be a system property (-D...) or a project
+property (-P...).
+
+Example:
+
+gradlew test -p lucene/test-framework --tests TestJvmInfo -Dtests.verbose=true -Druntime.java.home=/jvms/jdk14
+
+Note that an alternative JVM can also be made the "default" setting
+by adding it to (project-local) gradle.properties.
diff --git a/help/tests.txt b/help/tests.txt
index 2caef22..30b1f4a 100644
--- a/help/tests.txt
+++ b/help/tests.txt
@@ -140,18 +140,3 @@ Using these additional options will make the results more sparse, so it may be u
to increase the top-N count:
gradlew -p lucene/core test -Ptests.profile=true -Ptests.profile.count=100
-
-Testing against different JVMs
-------------------------------
-
-By default tests are executed with the same Java gradle is using internally.
-To run tests against a different Java version define a property called
-"runtime.java.home" or define an environment variable "RUNTIME_JAVA_HOME"
-pointing at the JDK installation folder.
-
-If property is used, it can be a system property (-D...) or a project
-property (-P...).
-
-Example:
-
-gradlew test -p lucene/test-framework --tests TestJvmInfo -Dtests.verbose=true -Druntime.java.home=/jvms/jdk14
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 29d5aba..d8bdf7f 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -12,7 +12,7 @@ System Requirements
API Changes
-* LUCENE-8474: RAMDirectory and associated deprecated classes have been
+* LUCENE-8474: RAMDirectory and associated deprecated classes have been
removed. (Dawid Weiss)
* LUCENE-3041: The deprecated Weight#extractTerms() method has been
@@ -58,8 +58,14 @@ API Changes
* LUCENE-9340: SimpleBindings#add(SortField) has been removed. (Alan Woodward)
+* LUCENE-9462: Fields without positions should still return MatchIterator.
+ (Alan Woodward, Dawid Weiss)
+
Improvements
+* LUCENE-9463: Query match region retrieval component, passage scoring and formatting
+ for building custom highlighters. (Alan Woodward, Dawid Weiss)
+
* LUCENE-9370: RegExp query is no longer lenient about inappropriate backslashes and
follows the Java Pattern policy for rejecting illegal syntax. (Mark Harwood)
@@ -107,6 +113,22 @@ Improvements
* LUCENE-9074: Introduce Slice Executor For Dynamic Runtime Execution Of Slices (Atri Sharma)
+* LUCENE-9280: Add an ability for field comparators to skip non-competitive documents.
+ Creating a TopFieldCollector with totalHitsThreshold less than Integer.MAX_VALUE
+ instructs Lucene to skip non-competitive documents whenever possible. For numeric
+ sort fields the skipping functionality works when the same field is indexed both
+ with doc values and points. In this case, there is an assumption that the same data is
+ stored in these points and doc values (Mayya Sharipova, Jim Ferenczi, Adrien Grand)
+
+* LUCENE-9313: Add SerbianAnalyzer based on the snowball stemmer. (Dragan Ivanovic)
+
+* LUCENE-9449: Enhance DocComparator to provide an iterator over competitive
+ documents when searching with "after". This iterator can quickly position
+ on the desired "after" document skipping all documents and segments before
+ "after". Also redesign numeric comparators to provide skipping functionality
+ by default. (Mayya Sharipova, Jim Ferenczi)
+
+
Bug fixes
* LUCENE-8663: NRTCachingDirectory.slowFileExists may open a file while
@@ -126,6 +148,10 @@ Bug fixes
Other
+* LUCENE-9312: Allow gradle builds against arbitrary JVMs. (Tomoko Uchida, Dawid Weiss)
+
+* LUCENE-9391: Upgrade HPPC to 0.8.2. (Haoyu Zhai)
+
* LUCENE-8768: Fix Javadocs build in Java 11. (Namgyu Kim)
* LUCENE-9092: upgrade randomizedtesting to 2.7.5 (Dawid Weiss)
@@ -141,6 +167,120 @@ Other
* LUCENE-9411: Fail complation on warnings, 9x gradle-only (Erick Erickson, Dawid Weiss)
Deserves mention here as well as Lucene CHANGES.txt since it affects both.
+* LUCENE-9433: Remove Ant support from trunk (Erick Erickson, Uwe Schindler et.al.)
+
+* LUCENE-9215: Replace checkJavaDocs.py with doclet (Robert Muir, Dawid Weiss, Uwe Schindler)
+
+* LUCENE-9497: Integrate Error Prone, a static analysis tool during compilation (Dawid Weiss, Varun Thacker)
+
+======================= Lucene 8.7.0 =======================
+
+API Changes
+---------------------
+
+* LUCENE-9437: Lucene's facet module's DocValuesOrdinalsReader.decode method
+ is now public, making it easier for applications to decode facet
+ ordinals into their corresponding labels (Ankur)
+
+* LUCENE-9515: IndexingChain now accepts individual primitives rather than a
+ DocumentsWriterPerThread instance in order to create a new DocConsumer.
+ (Simon Willnauer)
+
+New Features
+---------------------
+
+* LUCENE-9386: RegExpQuery added case insensitive matching option. (Mark Harwood)
+
+* LUCENE-8962: Add IndexWriter merge-on-refresh feature to selectively merge
+ small segments on getReader, subject to a configurable timeout, to improve
+ search performance by reducing the number of small segments for searching. (Simon Willnauer)
+
+* LUCENE-9484: Allow sorting an index after it was created. With SortingCodecReader, existing
+ unsorted segments can be wrapped and merged into a fresh index using IndexWriter#addIndices
+ API. (Simon Willnauer, Adrien Grand)
+
+Improvements
+---------------------
+
+* LUCENE-8574: Add a new ExpressionValueSource which will enforce only one value per name
+ per hit in dependencies, ExpressionFunctionValues will no longer
+ recompute already computed values (Haoyu Zhai)
+
+* LUCENE-9416: Fix CheckIndex to print an invalid non-zero norm as
+ unsigned long when detecting corruption.
+
+* LUCENE-9440: FieldInfo#checkConsistency called twice from Lucene50(60)FieldInfosFormat#read;
+ Removed the (redundant?) assert and do these checks for real. (Yauheni Putsykovich)
+
+* LUCENE-9446: In BooleanQuery rewrite, always remove MatchAllDocsQuery filter clauses
+ when possible. (Julie Tibshirani)
+
+* LUCENE-9501: Improve how Asserting* test classes handle singleton doc values.
+
+* LUCENE-9511: Include StoredFieldsWriter in DWPT accounting to ensure that it's
+ heap consumption is taken into account when IndexWriter stalls or should flush
+ DWPTs. (Simon Willnauer)
+
+* LUCENE-9514: Include TermVectorsWriter in DWPT accounting to ensure that it's
+ heap consumption is taken into account when IndexWriter stalls or should flush
+ DWPTs. (Simon Willnauer)
+
+Optimizations
+---------------------
+
+* LUCENE-9395: ConstantValuesSource now shares a single DoubleValues
+ instance across all segments (Tony Xu)
+
+* LUCENE-9447, LUCENE-9486: Stored fields now get higer compression ratios on
+ highly compressible data. (Adrien Grand)
+
+* LUCENE-9373: FunctionMatchQuery now accepts a "matchCost" optimization hint.
+ (Maxim Glazkov, David Smiley)
+
+Bug Fixes
+---------------------
+
+* LUCENE-9427: Fix a regression where the unified highlighter didn't produce
+ highlights on fuzzy queries that correspond to exact matches. (Julie Tibshirani)
+
+* LUCENE-9467: Fix NRTCachingDirectory to use Directory#fileLength to check if a file
+ already exists instead of opening an IndexInput on the file which might throw a AccessDeniedException
+ in some Directory implementations. (Simon Willnauer)
+
+* LUCENE-9501: Fix a bug in IndexSortSortedNumericDocValuesRangeQuery where it could violate the
+ DocIdSetIterator contract. (Julie Tibshirani)
+
+Documentation
+---------------------
+
+* LUCENE-9424: Add a performance warning to AttributeSource.captureState javadocs (Haoyu Zhai)
+
+
+Other
+---------------------
+
+* LUCENE-9292: Refactor BKD point configuration into its own class. (Ignacio Vera)
+
+* LUCENE-9470: Make TestXYMultiPolygonShapeQueries more resilient for CONTAINS queries. (Ignacio Vera)
+
+* LUCENE-9512: Move LockFactory stress test to be a unit/integration
+ test. (Uwe Schindler, Dawid Weiss, Robert Muir)
+
+======================= Lucene 8.6.2 =======================
+
+Bug Fixes
+---------------------
+* LUCENE-9478: Prevent DWPTDeleteQueue from referencing itself and leaking memory. The queue
+ passed an implicit this reference to the next queue instance on flush which leaked about 500byte
+ of memory on each full flush, commit or getReader call. (Simon Willnauer)
+
+======================= Lucene 8.6.1 =======================
+
+Bug Fixes
+---------------------
+* LUCENE-9443: The UnifiedHighlighter was closing the underlying reader when there were multiple term-vector fields.
+ This was a regression in 8.6.0. (David Smiley, Chris Beer)
+
======================= Lucene 8.6.0 =======================
API Changes
@@ -182,6 +322,10 @@ New Features
* LUCENE-7889: Grouping by range based on values from DoubleValuesSource and LongValuesSource
(Alan Woodward)
+* LUCENE-8962: Add IndexWriter merge-on-commit feature to selectively merge small segments on commit,
+ subject to a configurable timeout, to improve search performance by reducing the number of small
+ segments for searching (Michael Froh, Mike Sokolov, Mike Mccandless, Simon Willnauer)
+
Improvements
---------------------
* LUCENE-9276: Use same code-path for updateDocuments and updateDocument in IndexWriter and
@@ -201,12 +345,24 @@ Improvements
* LUCENE-9342: TotalHits' relation will be EQUAL_TO when the number of hits is lower than TopDocsColector's numHits
(Tomás Fernández Löbbe)
+* LUCENE-9353: Metadata of the terms dictionary moved to its own file, with the
+ `.tmd` extension. This allows checksums of metadata to be verified when
+ opening indices and helps save seeks when opening an index. (Adrien Grand)
+
* LUCENE-9359: SegmentInfos#readCommit now always returns a
CorruptIndexException if the content of the file is invalid. (Adrien Grand)
* LUCENE-9393: Make FunctionScoreQuery use ScoreMode.COMPLETE for creating the inner query weight when
ScoreMode.TOP_DOCS is requested. (Tomás Fernández Löbbe)
+* LUCENE-9392: Make FacetsConfig.DELIM_CHAR publicly accessible (Ankur Goel))
+
+* LUCENE-9397: UniformSplit supports encodable fields metadata. (Bruno Roustant)
+
+* LUCENE-9396: Improved truncation detection for points. (Adrien Grand, Robert Muir)
+
+* LUCENE-9402: Let MultiCollector handle minCompetitiveScore (Tomás Fernández Löbbe, Adrien Grand)
+
Optimizations
---------------------
@@ -230,6 +386,8 @@ Optimizations
* LUCENE-9087: Build always trees with full leaves and lower the default value for maxPointsPerLeafNode to 512.
(Ignacio Vera)
+* LUCENE-9148: Points now write their index in a separate file. (Adrien Grand)
+
Bug Fixes
---------------------
* LUCENE-9259: Fix wrong NGramFilterFactory argument name for preserveOriginal option (Paul Pazderski)
@@ -261,6 +419,16 @@ Bug Fixes
* LUCENE-9362: Fix equality check in ExpressionValueSource#rewrite. This fixes rewriting of inner value sources.
(Dmitry Emets)
+* LUCENE-9405: IndexWriter incorrectly calls closeMergeReaders twice when the merged segment is 100% deleted.
+ (Michael Froh, Simon Willnauer, Mike Mccandless, Mike Sokolov)
+
+* LUCENE-9400: Tessellator might build illegal polygons when several holes share the shame vertex. (Ignacio Vera)
+
+* LUCENE-9417: Tessellator might build illegal polygons when several holes share are connected to the same
+ vertex. (Ignacio Vera)
+
+* LUCENE-9418: Fix ordered intervals over interleaved terms (Alan Woodward)
+
Other
---------------------
@@ -293,6 +461,8 @@ Other
* LUCENE-9232: Fix or suppress 13 resource leak precommit warnings in lucene/replicator (Andras Salamon via Erick Erickson)
+* LUCENE-9398: Always keep BKD index off-heap. BKD reader does not implement Accountable any more. (Ignacio Vera)
+
Build
* Upgrade forbiddenapis to version 3.0.1. (Uwe Schindler)
@@ -302,6 +472,8 @@ Build
* LUCENE-9380: Fix auxiliary class warnings in Lucene (Erick Erickson)
+* LUCENE-9389: Enhance gradle logging calls validation: eliminate getMessage() (Andras Salamon via Erick Erickson)
+
======================= Lucene 8.5.2 =======================
Optimizations