You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2016/04/27 11:41:42 UTC
[1/2] lucene-solr:master: LUCENE-7257: PointValues aggregated stats
fail if the provided field does not have points on one of the leaves.
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 95322605b -> dbb28f3cb
refs/heads/master 9ce830d8f -> c7cdf2832
LUCENE-7257: PointValues aggregated stats fail if the provided field does not have points on one of the leaves.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c7cdf283
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c7cdf283
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c7cdf283
Branch: refs/heads/master
Commit: c7cdf2832bbfbc46d9bfa3cb5041a821fbb7be1d
Parents: 9ce830d
Author: Adrien Grand <jp...@gmail.com>
Authored: Wed Apr 27 11:40:32 2016 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Wed Apr 27 11:41:04 2016 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 ++
.../org/apache/lucene/index/PointValues.java | 38 +++++++++------
.../apache/lucene/index/TestPointValues.java | 50 ++++++++++++++++++++
3 files changed, 78 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c7cdf283/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index dc4cfcb..7a2cc03 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -104,6 +104,10 @@ Bug Fixes
* LUCENE-7232: Fixed InetAddressPoint.newPrefixQuery, which was generating an
incorrect query when the prefix length was not a multiple of 8. (Adrien Grand)
+* LUCENE-7257: Fixed PointValues#size(IndexReader, String), docCount,
+ minPackedValue and maxPackedValue to skip leaves that do not have points
+ rather than raising an IllegalStateException. (Adrien Grand)
+
Documentation
* LUCENE-7223: Improve XXXPoint javadocs to make it clear that you
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c7cdf283/lucene/core/src/java/org/apache/lucene/index/PointValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValues.java b/lucene/core/src/java/org/apache/lucene/index/PointValues.java
index a4fd323..528f971 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PointValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PointValues.java
@@ -87,43 +87,51 @@ public abstract class PointValues {
public static final int MAX_DIMENSIONS = BKDWriter.MAX_DIMS;
/** Return the cumulated number of points across all leaves of the given
- * {@link IndexReader}.
+ * {@link IndexReader}. Leaves that do not have points for the given field
+ * are ignored.
* @see PointValues#size(String) */
public static long size(IndexReader reader, String field) throws IOException {
long size = 0;
for (LeafReaderContext ctx : reader.leaves()) {
- PointValues values = ctx.reader().getPointValues();
- if (values != null) {
- size += values.size(field);
+ FieldInfo info = ctx.reader().getFieldInfos().fieldInfo(field);
+ if (info == null || info.getPointDimensionCount() == 0) {
+ continue;
}
+ PointValues values = ctx.reader().getPointValues();
+ size += values.size(field);
}
return size;
}
/** Return the cumulated number of docs that have points across all leaves
- * of the given {@link IndexReader}.
+ * of the given {@link IndexReader}. Leaves that do not have points for the
+ * given field are ignored.
* @see PointValues#getDocCount(String) */
public static int getDocCount(IndexReader reader, String field) throws IOException {
int count = 0;
for (LeafReaderContext ctx : reader.leaves()) {
- PointValues values = ctx.reader().getPointValues();
- if (values != null) {
- count += values.getDocCount(field);
+ FieldInfo info = ctx.reader().getFieldInfos().fieldInfo(field);
+ if (info == null || info.getPointDimensionCount() == 0) {
+ continue;
}
+ PointValues values = ctx.reader().getPointValues();
+ count += values.getDocCount(field);
}
return count;
}
/** Return the minimum packed values across all leaves of the given
- * {@link IndexReader}.
+ * {@link IndexReader}. Leaves that do not have points for the given field
+ * are ignored.
* @see PointValues#getMinPackedValue(String) */
public static byte[] getMinPackedValue(IndexReader reader, String field) throws IOException {
byte[] minValue = null;
for (LeafReaderContext ctx : reader.leaves()) {
- PointValues values = ctx.reader().getPointValues();
- if (values == null) {
+ FieldInfo info = ctx.reader().getFieldInfos().fieldInfo(field);
+ if (info == null || info.getPointDimensionCount() == 0) {
continue;
}
+ PointValues values = ctx.reader().getPointValues();
byte[] leafMinValue = values.getMinPackedValue(field);
if (leafMinValue == null) {
continue;
@@ -145,15 +153,17 @@ public abstract class PointValues {
}
/** Return the maximum packed values across all leaves of the given
- * {@link IndexReader}.
+ * {@link IndexReader}. Leaves that do not have points for the given field
+ * are ignored.
* @see PointValues#getMaxPackedValue(String) */
public static byte[] getMaxPackedValue(IndexReader reader, String field) throws IOException {
byte[] maxValue = null;
for (LeafReaderContext ctx : reader.leaves()) {
- PointValues values = ctx.reader().getPointValues();
- if (values == null) {
+ FieldInfo info = ctx.reader().getFieldInfos().fieldInfo(field);
+ if (info == null || info.getPointDimensionCount() == 0) {
continue;
}
+ PointValues values = ctx.reader().getPointValues();
byte[] leafMaxValue = values.getMaxPackedValue(field);
if (leafMaxValue == null) {
continue;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c7cdf283/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
index c7ca2dc..9f55ec3 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
@@ -30,6 +30,7 @@ import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
import org.apache.lucene.index.PointValues;
@@ -652,6 +653,55 @@ public class TestPointValues extends LuceneTestCase {
dir.close();
}
+ public void testMergedStatsEmptyReader() throws IOException {
+ IndexReader reader = new MultiReader();
+ assertNull(PointValues.getMinPackedValue(reader, "field"));
+ assertNull(PointValues.getMaxPackedValue(reader, "field"));
+ assertEquals(0, PointValues.getDocCount(reader, "field"));
+ assertEquals(0, PointValues.size(reader, "field"));
+ }
+
+ public void testMergedStatsOneSegmentWithoutPoints() throws IOException {
+ Directory dir = new RAMDirectory();
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null).setMergePolicy(NoMergePolicy.INSTANCE));
+ w.addDocument(new Document());
+ DirectoryReader.open(w).close();
+ Document doc = new Document();
+ doc.add(new IntPoint("field", Integer.MIN_VALUE));
+ w.addDocument(doc);
+ IndexReader reader = DirectoryReader.open(w);
+
+ assertArrayEquals(new byte[4], PointValues.getMinPackedValue(reader, "field"));
+ assertArrayEquals(new byte[4], PointValues.getMaxPackedValue(reader, "field"));
+ assertEquals(1, PointValues.getDocCount(reader, "field"));
+ assertEquals(1, PointValues.size(reader, "field"));
+
+ assertNull(PointValues.getMinPackedValue(reader, "field2"));
+ assertNull(PointValues.getMaxPackedValue(reader, "field2"));
+ assertEquals(0, PointValues.getDocCount(reader, "field2"));
+ assertEquals(0, PointValues.size(reader, "field2"));
+ }
+
+ public void testMergedStatsAllPointsDeleted() throws IOException {
+ Directory dir = new RAMDirectory();
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
+ w.addDocument(new Document());
+ Document doc = new Document();
+ doc.add(new IntPoint("field", Integer.MIN_VALUE));
+ doc.add(new StringField("delete", "yes", Store.NO));
+ w.addDocument(doc);
+ w.forceMerge(1);
+ w.deleteDocuments(new Term("delete", "yes"));
+ w.addDocument(new Document());
+ w.forceMerge(1);
+ IndexReader reader = DirectoryReader.open(w);
+
+ assertNull(PointValues.getMinPackedValue(reader, "field"));
+ assertNull(PointValues.getMaxPackedValue(reader, "field"));
+ assertEquals(0, PointValues.getDocCount(reader, "field"));
+ assertEquals(0, PointValues.size(reader, "field"));
+ }
+
public void testMergedStats() throws IOException {
final int iters = atLeast(3);
for (int iter = 0; iter < iters; ++iter) {
[2/2] lucene-solr:branch_6x: LUCENE-7257: PointValues aggregated
stats fail if the provided field does not have points on one of the leaves.
Posted by jp...@apache.org.
LUCENE-7257: PointValues aggregated stats fail if the provided field does not have points on one of the leaves.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/dbb28f3c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/dbb28f3c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/dbb28f3c
Branch: refs/heads/branch_6x
Commit: dbb28f3cb431cf71133c2da76172a9cae480e3d5
Parents: 9532260
Author: Adrien Grand <jp...@gmail.com>
Authored: Wed Apr 27 11:40:32 2016 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Wed Apr 27 11:41:18 2016 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 ++
.../org/apache/lucene/index/PointValues.java | 38 +++++++++------
.../apache/lucene/index/TestPointValues.java | 50 ++++++++++++++++++++
3 files changed, 78 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/dbb28f3c/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 5fce253..d66dfc6 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -97,6 +97,10 @@ Bug Fixes
* LUCENE-7232: Fixed InetAddressPoint.newPrefixQuery, which was generating an
incorrect query when the prefix length was not a multiple of 8. (Adrien Grand)
+* LUCENE-7257: Fixed PointValues#size(IndexReader, String), docCount,
+ minPackedValue and maxPackedValue to skip leaves that do not have points
+ rather than raising an IllegalStateException. (Adrien Grand)
+
Documentation
* LUCENE-7223: Improve XXXPoint javadocs to make it clear that you
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/dbb28f3c/lucene/core/src/java/org/apache/lucene/index/PointValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValues.java b/lucene/core/src/java/org/apache/lucene/index/PointValues.java
index a4fd323..528f971 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PointValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PointValues.java
@@ -87,43 +87,51 @@ public abstract class PointValues {
public static final int MAX_DIMENSIONS = BKDWriter.MAX_DIMS;
/** Return the cumulated number of points across all leaves of the given
- * {@link IndexReader}.
+ * {@link IndexReader}. Leaves that do not have points for the given field
+ * are ignored.
* @see PointValues#size(String) */
public static long size(IndexReader reader, String field) throws IOException {
long size = 0;
for (LeafReaderContext ctx : reader.leaves()) {
- PointValues values = ctx.reader().getPointValues();
- if (values != null) {
- size += values.size(field);
+ FieldInfo info = ctx.reader().getFieldInfos().fieldInfo(field);
+ if (info == null || info.getPointDimensionCount() == 0) {
+ continue;
}
+ PointValues values = ctx.reader().getPointValues();
+ size += values.size(field);
}
return size;
}
/** Return the cumulated number of docs that have points across all leaves
- * of the given {@link IndexReader}.
+ * of the given {@link IndexReader}. Leaves that do not have points for the
+ * given field are ignored.
* @see PointValues#getDocCount(String) */
public static int getDocCount(IndexReader reader, String field) throws IOException {
int count = 0;
for (LeafReaderContext ctx : reader.leaves()) {
- PointValues values = ctx.reader().getPointValues();
- if (values != null) {
- count += values.getDocCount(field);
+ FieldInfo info = ctx.reader().getFieldInfos().fieldInfo(field);
+ if (info == null || info.getPointDimensionCount() == 0) {
+ continue;
}
+ PointValues values = ctx.reader().getPointValues();
+ count += values.getDocCount(field);
}
return count;
}
/** Return the minimum packed values across all leaves of the given
- * {@link IndexReader}.
+ * {@link IndexReader}. Leaves that do not have points for the given field
+ * are ignored.
* @see PointValues#getMinPackedValue(String) */
public static byte[] getMinPackedValue(IndexReader reader, String field) throws IOException {
byte[] minValue = null;
for (LeafReaderContext ctx : reader.leaves()) {
- PointValues values = ctx.reader().getPointValues();
- if (values == null) {
+ FieldInfo info = ctx.reader().getFieldInfos().fieldInfo(field);
+ if (info == null || info.getPointDimensionCount() == 0) {
continue;
}
+ PointValues values = ctx.reader().getPointValues();
byte[] leafMinValue = values.getMinPackedValue(field);
if (leafMinValue == null) {
continue;
@@ -145,15 +153,17 @@ public abstract class PointValues {
}
/** Return the maximum packed values across all leaves of the given
- * {@link IndexReader}.
+ * {@link IndexReader}. Leaves that do not have points for the given field
+ * are ignored.
* @see PointValues#getMaxPackedValue(String) */
public static byte[] getMaxPackedValue(IndexReader reader, String field) throws IOException {
byte[] maxValue = null;
for (LeafReaderContext ctx : reader.leaves()) {
- PointValues values = ctx.reader().getPointValues();
- if (values == null) {
+ FieldInfo info = ctx.reader().getFieldInfos().fieldInfo(field);
+ if (info == null || info.getPointDimensionCount() == 0) {
continue;
}
+ PointValues values = ctx.reader().getPointValues();
byte[] leafMaxValue = values.getMaxPackedValue(field);
if (leafMaxValue == null) {
continue;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/dbb28f3c/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
index c7ca2dc..9f55ec3 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
@@ -30,6 +30,7 @@ import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
import org.apache.lucene.index.PointValues;
@@ -652,6 +653,55 @@ public class TestPointValues extends LuceneTestCase {
dir.close();
}
+ public void testMergedStatsEmptyReader() throws IOException {
+ IndexReader reader = new MultiReader();
+ assertNull(PointValues.getMinPackedValue(reader, "field"));
+ assertNull(PointValues.getMaxPackedValue(reader, "field"));
+ assertEquals(0, PointValues.getDocCount(reader, "field"));
+ assertEquals(0, PointValues.size(reader, "field"));
+ }
+
+ public void testMergedStatsOneSegmentWithoutPoints() throws IOException {
+ Directory dir = new RAMDirectory();
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null).setMergePolicy(NoMergePolicy.INSTANCE));
+ w.addDocument(new Document());
+ DirectoryReader.open(w).close();
+ Document doc = new Document();
+ doc.add(new IntPoint("field", Integer.MIN_VALUE));
+ w.addDocument(doc);
+ IndexReader reader = DirectoryReader.open(w);
+
+ assertArrayEquals(new byte[4], PointValues.getMinPackedValue(reader, "field"));
+ assertArrayEquals(new byte[4], PointValues.getMaxPackedValue(reader, "field"));
+ assertEquals(1, PointValues.getDocCount(reader, "field"));
+ assertEquals(1, PointValues.size(reader, "field"));
+
+ assertNull(PointValues.getMinPackedValue(reader, "field2"));
+ assertNull(PointValues.getMaxPackedValue(reader, "field2"));
+ assertEquals(0, PointValues.getDocCount(reader, "field2"));
+ assertEquals(0, PointValues.size(reader, "field2"));
+ }
+
+ public void testMergedStatsAllPointsDeleted() throws IOException {
+ Directory dir = new RAMDirectory();
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
+ w.addDocument(new Document());
+ Document doc = new Document();
+ doc.add(new IntPoint("field", Integer.MIN_VALUE));
+ doc.add(new StringField("delete", "yes", Store.NO));
+ w.addDocument(doc);
+ w.forceMerge(1);
+ w.deleteDocuments(new Term("delete", "yes"));
+ w.addDocument(new Document());
+ w.forceMerge(1);
+ IndexReader reader = DirectoryReader.open(w);
+
+ assertNull(PointValues.getMinPackedValue(reader, "field"));
+ assertNull(PointValues.getMaxPackedValue(reader, "field"));
+ assertEquals(0, PointValues.getDocCount(reader, "field"));
+ assertEquals(0, PointValues.size(reader, "field"));
+ }
+
public void testMergedStats() throws IOException {
final int iters = atLeast(3);
for (int iter = 0; iter < iters; ++iter) {