You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2022/01/03 14:13:32 UTC
[lucene] branch branch_9x updated (837e163 -> 01f5e7b)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a change to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git.
from 837e163 LUCENE-10349: Cleanup WordListLoader to use try-with-resources and make the default stop words unmodifiable (#577)
new da3992b Add back-compat indices for 9.0.0
new 01f5e7b LUCENE-10346: Specially treat SingletonSortedNumericDocValues in FastTaxonomyFacetCounts#countAll() (#574)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
lucene/CHANGES.txt | 2 +
.../backward_index/TestBackwardsCompatibility.java | 43 +++++++++++++--------
.../lucene/backward_index/dvupdates.8.0.0.zip | Bin 13396 -> 0 bytes
.../lucene/backward_index/dvupdates.9.0.0.zip | Bin 0 -> 14471 bytes
.../apache/lucene/backward_index/empty.8.0.0.zip | Bin 376 -> 0 bytes
.../apache/lucene/backward_index/empty.9.0.0.zip | Bin 0 -> 376 bytes
.../lucene/backward_index/index.9.0.0-cfs.zip | Bin 0 -> 16674 bytes
.../lucene/backward_index/index.9.0.0-nocfs.zip | Bin 0 -> 16665 bytes
.../lucene/backward_index/moreterms.8.0.0.zip | Bin 82427 -> 0 bytes
.../lucene/backward_index/moreterms.9.0.0.zip | Bin 0 -> 141561 bytes
.../apache/lucene/backward_index/sorted.9.0.0.zip | Bin 0 -> 132016 bytes
.../facet/taxonomy/FastTaxonomyFacetCounts.java | 13 +++++++
12 files changed, 42 insertions(+), 16 deletions(-)
delete mode 100644 lucene/backward-codecs/src/test/org/apache/lucene/backward_index/dvupdates.8.0.0.zip
create mode 100644 lucene/backward-codecs/src/test/org/apache/lucene/backward_index/dvupdates.9.0.0.zip
delete mode 100644 lucene/backward-codecs/src/test/org/apache/lucene/backward_index/empty.8.0.0.zip
create mode 100644 lucene/backward-codecs/src/test/org/apache/lucene/backward_index/empty.9.0.0.zip
create mode 100644 lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.9.0.0-cfs.zip
create mode 100644 lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.9.0.0-nocfs.zip
delete mode 100644 lucene/backward-codecs/src/test/org/apache/lucene/backward_index/moreterms.8.0.0.zip
create mode 100644 lucene/backward-codecs/src/test/org/apache/lucene/backward_index/moreterms.9.0.0.zip
create mode 100644 lucene/backward-codecs/src/test/org/apache/lucene/backward_index/sorted.9.0.0.zip
[lucene] 01/02: Add back-compat indices for 9.0.0
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit da3992b16fc3abd43fc2e3a19c8bd9630dcae572
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Wed Dec 8 13:42:40 2021 +0100
Add back-compat indices for 9.0.0
---
.../backward_index/TestBackwardsCompatibility.java | 43 +++++++++++++--------
.../lucene/backward_index/dvupdates.8.0.0.zip | Bin 13396 -> 0 bytes
.../lucene/backward_index/dvupdates.9.0.0.zip | Bin 0 -> 14471 bytes
.../apache/lucene/backward_index/empty.8.0.0.zip | Bin 376 -> 0 bytes
.../apache/lucene/backward_index/empty.9.0.0.zip | Bin 0 -> 376 bytes
.../lucene/backward_index/index.9.0.0-cfs.zip | Bin 0 -> 16674 bytes
.../lucene/backward_index/index.9.0.0-nocfs.zip | Bin 0 -> 16665 bytes
.../lucene/backward_index/moreterms.8.0.0.zip | Bin 82427 -> 0 bytes
.../lucene/backward_index/moreterms.9.0.0.zip | Bin 0 -> 141561 bytes
.../apache/lucene/backward_index/sorted.9.0.0.zip | Bin 0 -> 132016 bytes
10 files changed, 27 insertions(+), 16 deletions(-)
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
index 7c177a9..2ae164c 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
@@ -382,7 +382,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
"8.11.0-cfs",
"8.11.0-nocfs",
"8.11.1-cfs",
- "8.11.1-nocfs"
+ "8.11.1-nocfs",
+ "9.0.0-cfs",
+ "9.0.0-nocfs"
};
public static String[] getOldNames() {
@@ -413,7 +415,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
"sorted.8.9.0",
"sorted.8.10.1",
"sorted.8.11.0",
- "sorted.8.11.1"
+ "sorted.8.11.1",
+ "sorted.9.0.0"
};
public static String[] getOldSortedNames() {
@@ -1784,7 +1787,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
}
- public static final String emptyIndex = "empty.8.0.0.zip";
+ public static final String emptyIndex = "empty.9.0.0.zip";
public void testUpgradeEmptyOldIndex() throws Exception {
Path oldIndexDir = createTempDir("emptyIndex");
@@ -1793,12 +1796,12 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
newIndexUpgrader(dir).upgrade();
- checkAllSegmentsUpgraded(dir, 8);
+ checkAllSegmentsUpgraded(dir, 9);
dir.close();
}
- public static final String moreTermsIndex = "moreterms.8.0.0.zip";
+ public static final String moreTermsIndex = "moreterms.9.0.0.zip";
public void testMoreTerms() throws Exception {
Path oldIndexDir = createTempDir("moreterms");
@@ -1814,7 +1817,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
dir.close();
}
- public static final String dvUpdatesIndex = "dvupdates.8.0.0.zip";
+ public static final String dvUpdatesIndex = "dvupdates.9.0.0.zip";
private void assertNumericDocValues(LeafReader r, String f, String cf) throws IOException {
NumericDocValues ndvf = r.getNumericDocValues(f);
@@ -2099,16 +2102,24 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
for (String name : oldNames) {
Directory directory = oldIndexDirs.get(name);
IndexCommit commit = DirectoryReader.listCommits(directory).get(0);
- IndexFormatTooOldException ex =
- expectThrows(
- IndexFormatTooOldException.class,
- () -> StandardDirectoryReader.open(commit, Version.LATEST.major, null));
- assertTrue(
- ex.getMessage()
- .contains(
- "only supports reading from version " + Version.LATEST.major + " upwards."));
- // now open with allowed min version
- StandardDirectoryReader.open(commit, Version.MIN_SUPPORTED_MAJOR, null).close();
+
+ final int createdMajor;
+ // No exception when opening with the allowed min version
+ try (IndexReader reader =
+ StandardDirectoryReader.open(commit, Version.MIN_SUPPORTED_MAJOR, null)) {
+ createdMajor = reader.leaves().get(0).reader().getMetaData().getCreatedVersionMajor();
+ }
+
+ if (createdMajor < Version.LATEST.major) {
+ IndexFormatTooOldException ex =
+ expectThrows(
+ IndexFormatTooOldException.class,
+ () -> StandardDirectoryReader.open(commit, Version.LATEST.major, null));
+ assertTrue(
+ ex.getMessage()
+ .contains(
+ "only supports reading from version " + Version.LATEST.major + " upwards."));
+ }
}
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/dvupdates.8.0.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/dvupdates.8.0.0.zip
deleted file mode 100644
index 38c5428..0000000
Binary files a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/dvupdates.8.0.0.zip and /dev/null differ
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/dvupdates.9.0.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/dvupdates.9.0.0.zip
new file mode 100644
index 0000000..1416538
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/dvupdates.9.0.0.zip differ
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/empty.8.0.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/empty.8.0.0.zip
deleted file mode 100644
index 91a9e92..0000000
Binary files a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/empty.8.0.0.zip and /dev/null differ
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/empty.9.0.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/empty.9.0.0.zip
new file mode 100644
index 0000000..f898ef2
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/empty.9.0.0.zip differ
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.9.0.0-cfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.9.0.0-cfs.zip
new file mode 100644
index 0000000..cc0a072
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.9.0.0-cfs.zip differ
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.9.0.0-nocfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.9.0.0-nocfs.zip
new file mode 100644
index 0000000..160cba2
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.9.0.0-nocfs.zip differ
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/moreterms.8.0.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/moreterms.8.0.0.zip
deleted file mode 100644
index 82b0f14..0000000
Binary files a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/moreterms.8.0.0.zip and /dev/null differ
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/moreterms.9.0.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/moreterms.9.0.0.zip
new file mode 100644
index 0000000..c83401f
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/moreterms.9.0.0.zip differ
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/sorted.9.0.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/sorted.9.0.0.zip
new file mode 100644
index 0000000..d70dac0
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/sorted.9.0.0.zip differ
[lucene] 02/02: LUCENE-10346: Specially treat SingletonSortedNumericDocValues in FastTaxonomyFacetCounts#countAll() (#574)
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 01f5e7bb7b4f5efb5330ab97896008c83daef657
Author: gf2121 <52...@users.noreply.github.com>
AuthorDate: Mon Jan 3 21:44:05 2022 +0800
LUCENE-10346: Specially treat SingletonSortedNumericDocValues in FastTaxonomyFacetCounts#countAll() (#574)
---
lucene/CHANGES.txt | 2 ++
.../lucene/facet/taxonomy/FastTaxonomyFacetCounts.java | 13 +++++++++++++
2 files changed, 15 insertions(+)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 2c87919..d11c51e 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -106,6 +106,8 @@ Optimizations
asked to -- typically never. This fixes a performance regression since 7.3 LUCENE-8099 when some
older boosting queries were replaced with this. (David Smiley)
+* LUCENE-10346: Optimize facet counting for single-valued TaxonomyFacetCounts. (Guo Feng)
+
Bug Fixes
---------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
index 668b773..0d8b26e 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
@@ -23,8 +23,10 @@ import org.apache.lucene.facet.FacetUtils;
import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
import org.apache.lucene.facet.FacetsConfig;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.search.ConjunctionUtils;
import org.apache.lucene.search.DocIdSetIterator;
@@ -98,6 +100,17 @@ public class FastTaxonomyFacetCounts extends IntTaxonomyFacets {
Bits liveDocs = context.reader().getLiveDocs();
+ NumericDocValues ndv = DocValues.unwrapSingleton(dv);
+ if (ndv != null) {
+ for (int doc = ndv.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = ndv.nextDoc()) {
+ if (liveDocs != null && liveDocs.get(doc) == false) {
+ continue;
+ }
+ increment((int) ndv.longValue());
+ }
+ continue;
+ }
+
for (int doc = dv.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = dv.nextDoc()) {
if (liveDocs != null && liveDocs.get(doc) == false) {
continue;