You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/02/15 23:58:28 UTC
svn commit: r1446801 [1/2] - in /lucene/dev/branches/lucene4765: ./
dev-tools/ dev-tools/maven/ lucene/ lucene/core/
lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/java/org/apache/lucene/search/
lucene/core/src/java/org/apache/lucene/sto...
Author: rmuir
Date: Fri Feb 15 22:58:27 2013
New Revision: 1446801
URL: http://svn.apache.org/r1446801
Log:
Merged /lucene/dev/trunk:r1445907-1446753
Added:
lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSort2.java
- copied unchanged from r1446753, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestSort2.java
lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java
- copied unchanged from r1446753, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java
lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
- copied unchanged from r1446753, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
- copied unchanged from r1446753, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java
- copied unchanged from r1446753, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java
lucene/dev/branches/lucene4765/solr/example/etc/create-solrtest.keystore.sh
- copied unchanged from r1446753, lucene/dev/trunk/solr/example/etc/create-solrtest.keystore.sh
lucene/dev/branches/lucene4765/solr/example/etc/solrtest.keystore
- copied unchanged from r1446753, lucene/dev/trunk/solr/example/etc/solrtest.keystore
lucene/dev/branches/lucene4765/solr/licenses/httpclient-4.2.3.jar.sha1
- copied unchanged from r1446753, lucene/dev/trunk/solr/licenses/httpclient-4.2.3.jar.sha1
lucene/dev/branches/lucene4765/solr/licenses/httpcore-4.2.2.jar.sha1
- copied unchanged from r1446753, lucene/dev/trunk/solr/licenses/httpcore-4.2.2.jar.sha1
lucene/dev/branches/lucene4765/solr/licenses/httpmime-4.2.3.jar.sha1
- copied unchanged from r1446753, lucene/dev/trunk/solr/licenses/httpmime-4.2.3.jar.sha1
Removed:
lucene/dev/branches/lucene4765/solr/licenses/httpclient-4.1.3.jar.sha1
lucene/dev/branches/lucene4765/solr/licenses/httpcore-4.1.4.jar.sha1
lucene/dev/branches/lucene4765/solr/licenses/httpmime-4.1.3.jar.sha1
Modified:
lucene/dev/branches/lucene4765/ (props changed)
lucene/dev/branches/lucene4765/dev-tools/ (props changed)
lucene/dev/branches/lucene4765/dev-tools/maven/pom.xml.template
lucene/dev/branches/lucene4765/lucene/ (props changed)
lucene/dev/branches/lucene4765/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene4765/lucene/core/ (props changed)
lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java
lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java
lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSort.java
lucene/dev/branches/lucene4765/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java
lucene/dev/branches/lucene4765/lucene/spatial/ (props changed)
lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java
lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java
lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java
lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java
lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java
lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java
lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java
lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeTest.java
lucene/dev/branches/lucene4765/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
lucene/dev/branches/lucene4765/lucene/tools/ (props changed)
lucene/dev/branches/lucene4765/lucene/tools/junit4/tests.policy
lucene/dev/branches/lucene4765/solr/ (props changed)
lucene/dev/branches/lucene4765/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene4765/solr/core/ (props changed)
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/SolrCore.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
lucene/dev/branches/lucene4765/solr/example/ (props changed)
lucene/dev/branches/lucene4765/solr/example/etc/jetty.xml
lucene/dev/branches/lucene4765/solr/licenses/ (props changed)
lucene/dev/branches/lucene4765/solr/solrj/ (props changed)
lucene/dev/branches/lucene4765/solr/solrj/ivy.xml
lucene/dev/branches/lucene4765/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
lucene/dev/branches/lucene4765/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
lucene/dev/branches/lucene4765/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
lucene/dev/branches/lucene4765/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClientUtilTest.java
lucene/dev/branches/lucene4765/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrServerTest.java
lucene/dev/branches/lucene4765/solr/test-framework/ (props changed)
lucene/dev/branches/lucene4765/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java
Modified: lucene/dev/branches/lucene4765/dev-tools/maven/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/dev-tools/maven/pom.xml.template?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/dev-tools/maven/pom.xml.template (original)
+++ lucene/dev/branches/lucene4765/dev-tools/maven/pom.xml.template Fri Feb 15 22:58:27 2013
@@ -48,7 +48,7 @@
<jetty.version>8.1.8.v20121106</jetty.version>
<slf4j.version>1.6.4</slf4j.version>
<tika.version>1.2</tika.version>
- <httpcomponents.version>4.1.3</httpcomponents.version>
+ <httpcomponents.version>4.2.3</httpcomponents.version>
<!-- RandomizedTesting library system properties -->
<tests.iters>1</tests.iters>
Modified: lucene/dev/branches/lucene4765/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/CHANGES.txt?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene4765/lucene/CHANGES.txt Fri Feb 15 22:58:27 2013
@@ -168,6 +168,9 @@ New Features
is faster than others, however consumes much more RAM.
(Michael McCandless, Shai Erera)
+* LUCENE-4778: Add a getter for the delegate in RateLimitedDirectoryWrapper.
+ (Mark Miller)
+
API Changes
* LUCENE-4709: FacetResultNode no longer has a residue field. (Shai Erera)
@@ -216,6 +219,19 @@ Bug Fixes
cases, for example if you had an index with more than 260M documents and a
VAR_INT field. (Simon Willnauer, Adrien Grand, Mike McCandless, Robert Muir)
+* LUCENE-4775: Remove SegmentInfo.sizeInBytes() and make
+ MergePolicy.OneMerge.totalBytesSize thread safe (Josh Bronson via
+ Robert Muir, Mike McCandless)
+
+* LUCENE-4770: If spatial's TermQueryPrefixTreeStrategy was used to search
+ indexed non-point shapes, then there was an edge case where a query should
+ find a shape but it didn't. The fix is the removal of an optimization that
+ simplifies some leaf cells into a parent. The index data for such a field is
+ now ~20% larger. This optimization is still done for the query shape, and for
+ indexed data for RecursivePrefixTreeStrategy. Furthermore, this optimization
+ is enhanced to roll up beyond the bottom cell level. (David Smiley,
+ Florian Schilling)
+
Documentation
* LUCENE-4718: Fixed documentation of oal.queryparser.classic.
Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Fri Feb 15 22:58:27 2013
@@ -520,7 +520,7 @@ class DocumentsWriterPerThread {
}
if (infoStream.isEnabled("DWPT")) {
- final double newSegmentSize = segmentInfo.sizeInBytes()/1024./1024.;
+ final double newSegmentSize = segmentInfoPerCommit.sizeInBytes()/1024./1024.;
infoStream.message("DWPT", "flushed: segment=" + segmentInfo.name +
" ramUsed=" + nf.format(startMBUsed) + " MB" +
" newFlushedSize(includes docstores)=" + nf.format(newSegmentSize) + " MB" +
@@ -557,7 +557,7 @@ class DocumentsWriterPerThread {
IndexWriter.setDiagnostics(newSegment.info, "flush");
- IOContext context = new IOContext(new FlushInfo(newSegment.info.getDocCount(), newSegment.info.sizeInBytes()));
+ IOContext context = new IOContext(new FlushInfo(newSegment.info.getDocCount(), newSegment.sizeInBytes()));
boolean success = false;
try {
Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Fri Feb 15 22:58:27 2013
@@ -2301,7 +2301,7 @@ public class IndexWriter implements Clos
infoStream.message("IW", "addIndexes: process segment origName=" + info.info.name + " newName=" + newSegName + " info=" + info);
}
- IOContext context = new IOContext(new MergeInfo(info.info.getDocCount(), info.info.sizeInBytes(), true, -1));
+ IOContext context = new IOContext(new MergeInfo(info.info.getDocCount(), info.sizeInBytes(), true, -1));
for(FieldInfo fi : getFieldInfos(info.info)) {
globalFieldNumberMap.addOrGet(fi.name, fi.number, fi.getDocValuesType());
@@ -3458,7 +3458,8 @@ public class IndexWriter implements Clos
final int delCount = numDeletedDocs(info);
assert delCount <= info.info.getDocCount();
final double delRatio = ((double) delCount)/info.info.getDocCount();
- merge.estimatedMergeBytes += info.info.sizeInBytes() * (1.0 - delRatio);
+ merge.estimatedMergeBytes += info.sizeInBytes() * (1.0 - delRatio);
+ merge.totalMergeBytes += info.sizeInBytes();
}
}
}
@@ -3759,7 +3760,7 @@ public class IndexWriter implements Clos
// lost...
if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", String.format(Locale.ROOT, "merged segment size=%.3f MB vs estimate=%.3f MB", merge.info.info.sizeInBytes()/1024./1024., merge.estimatedMergeBytes/1024/1024.));
+ infoStream.message("IW", String.format(Locale.ROOT, "merged segment size=%.3f MB vs estimate=%.3f MB", merge.info.sizeInBytes()/1024./1024., merge.estimatedMergeBytes/1024/1024.));
}
final IndexReaderWarmer mergedSegmentWarmer = config.getMergedSegmentWarmer();
Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java Fri Feb 15 22:58:27 2013
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -74,7 +73,11 @@ public abstract class MergePolicy implem
int maxNumSegments = -1; // used by IndexWriter
/** Estimated size in bytes of the merged segment. */
- public long estimatedMergeBytes; // used by IndexWriter
+ public volatile long estimatedMergeBytes; // used by IndexWriter
+
+ // Sum of sizeInBytes of all SegmentInfos; set by IW.mergeInit
+ volatile long totalMergeBytes;
+
List<SegmentReader> readers; // used by IndexWriter
/** Segments to be merged. */
@@ -187,14 +190,12 @@ public abstract class MergePolicy implem
/**
* Returns the total size in bytes of this merge. Note that this does not
- * indicate the size of the merged segment, but the input total size.
- * */
+ * indicate the size of the merged segment, but the
+ * input total size. This is only set once the merge is
+ * initialized by IndexWriter.
+ */
public long totalBytesSize() throws IOException {
- long total = 0;
- for (SegmentInfoPerCommit info : segments) {
- total += info.info.sizeInBytes();
- }
- return total;
+ return totalMergeBytes;
}
/**
Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java Fri Feb 15 22:58:27 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
*/
-import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -57,8 +56,6 @@ public final class SegmentInfo {
private boolean isCompoundFile;
- private volatile long sizeInBytes = -1; // total byte size of all files (computed on demand)
-
private Codec codec;
private Map<String,String> diagnostics;
@@ -101,23 +98,6 @@ public final class SegmentInfo {
}
/**
- * Returns total size in bytes of all of files used by
- * this segment. Note that this will not include any live
- * docs for the segment; to include that use {@link
- * SegmentInfoPerCommit#sizeInBytes()} instead.
- */
- public long sizeInBytes() throws IOException {
- if (sizeInBytes == -1) {
- long sum = 0;
- for (final String fileName : files()) {
- sum += dir.fileLength(fileName);
- }
- sizeInBytes = sum;
- }
- return sizeInBytes;
- }
-
- /**
* Mark whether this segment is stored as a compound file.
*
* @param isCompoundFile true if this is a compound file;
@@ -254,7 +234,6 @@ public final class SegmentInfo {
public void setFiles(Set<String> files) {
checkFileNames(files);
setFiles = files;
- sizeInBytes = -1;
}
/** Add these files to the set of files written for this
@@ -262,7 +241,6 @@ public final class SegmentInfo {
public void addFiles(Collection<String> files) {
checkFileNames(files);
setFiles.addAll(files);
- sizeInBytes = -1;
}
/** Add this file to the set of files written for this
@@ -270,7 +248,6 @@ public final class SegmentInfo {
public void addFile(String file) {
checkFileNames(Collections.singleton(file));
setFiles.add(file);
- sizeInBytes = -1;
}
private void checkFileNames(Collection<String> files) {
Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java Fri Feb 15 22:58:27 2013
@@ -374,7 +374,7 @@ public class TieredMergePolicy extends M
for(int idx = tooBigCount; idx<infosSorted.size(); idx++) {
final SegmentInfoPerCommit info = infosSorted.get(idx);
if (merging.contains(info)) {
- mergingBytes += info.info.sizeInBytes();
+ mergingBytes += info.sizeInBytes();
} else if (!toBeMerged.contains(info)) {
eligible.add(info);
}
@@ -470,7 +470,7 @@ public class TieredMergePolicy extends M
final long segBytes = size(info);
totAfterMergeBytes += segBytes;
totAfterMergeBytesFloored += floorSize(segBytes);
- totBeforeMergeBytes += info.info.sizeInBytes();
+ totBeforeMergeBytes += info.sizeInBytes();
}
// Measure "skew" of the merge, which can range
@@ -670,7 +670,7 @@ public class TieredMergePolicy extends M
// Segment size in bytes, pro-rated by % deleted
private long size(SegmentInfoPerCommit info) throws IOException {
- final long byteSize = info.info.sizeInBytes();
+ final long byteSize = info.sizeInBytes();
final int delCount = writer.get().numDeletedDocs(info);
final double delRatio = (info.info.getDocCount() <= 0 ? 0.0f : ((double)delCount / (double)info.info.getDocCount()));
assert delRatio <= 1.0;
Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/search/FieldCache.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/search/FieldCache.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/search/FieldCache.java Fri Feb 15 22:58:27 2013
@@ -91,7 +91,7 @@ public interface FieldCache {
};
}
- /** Field values as 32-bit signed long integers */
+ /** Field values as 64-bit signed long integers */
public static abstract class Longs {
/** Return an long representation of this field's value. */
public abstract long get(int docID);
Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java Fri Feb 15 22:58:27 2013
@@ -40,6 +40,10 @@ public final class RateLimitedDirectoryW
this.delegate = wrapped;
}
+ public Directory getDelegate() {
+ return delegate;
+ }
+
@Override
public String[] listAll() throws IOException {
ensureOpen();
Modified: lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java Fri Feb 15 22:58:27 2013
@@ -324,4 +324,40 @@ public class TestConcurrentMergeSchedule
w.close(false);
dir.close();
}
+
+
+ private static class TrackingCMS extends ConcurrentMergeScheduler {
+ long totMergedBytes;
+
+ public TrackingCMS() {
+ setMaxMergeCount(5);
+ setMaxThreadCount(5);
+ }
+
+ @Override
+ public void doMerge(MergePolicy.OneMerge merge) throws IOException {
+ totMergedBytes += merge.totalBytesSize();
+ super.doMerge(merge);
+ }
+ }
+
+ public void testTotalBytesSize() throws Exception {
+ Directory d = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ iwc.setMaxBufferedDocs(5);
+ iwc.setMergeScheduler(new TrackingCMS());
+ RandomIndexWriter w = new RandomIndexWriter(random(), d);
+ for(int i=0;i<100000;i++) {
+ Document doc = new Document();
+ doc.add(newStringField("id", ""+i, Field.Store.NO));
+ doc.add(newTextField("field", "here is some text", Field.Store.NO));
+ w.addDocument(doc);
+
+ if (random().nextBoolean()) {
+ w.deleteDocuments(new Term("id", ""+random().nextInt(i+1)));
+ }
+ }
+ w.close();
+ d.close();
+ }
}
Modified: lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSort.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSort.java Fri Feb 15 22:58:27 2013
@@ -20,11 +20,6 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.ArrayList;
import java.util.BitSet;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@@ -44,7 +39,6 @@ import org.apache.lucene.index.Directory
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.RandomIndexWriter;
@@ -53,13 +47,10 @@ import org.apache.lucene.index.StoredDoc
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.FieldValueHitQueue.Entry;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.DocIdBitSet;
-import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util._TestUtil;
@@ -351,11 +342,6 @@ public class TestSort extends LuceneTest
return getIndex(false, true);
}
- private IndexSearcher getEmptyIndex()
- throws IOException {
- return getIndex(false, false);
- }
-
// Set to true if the DV "string" field is indexed as a
// sorted source:
private boolean dvStringSorted;
@@ -392,64 +378,6 @@ public class TestSort extends LuceneTest
super.tearDown();
}
- // test the sorts by score and document number
- public void testBuiltInSorts() throws Exception {
- sort = new Sort();
- assertMatches(full, queryX, sort, "ACEGI");
- assertMatches(full, queryY, sort, "BDFHJ");
-
- sort.setSort(SortField.FIELD_DOC);
- assertMatches(full, queryX, sort, "ACEGI");
- assertMatches(full, queryY, sort, "BDFHJ");
- }
-
- // test sorts where the type of field is specified
- public void testTypedSort() throws Exception {
- sort.setSort(new SortField("int", SortField.Type.INT), SortField.FIELD_DOC);
- assertMatches(full, queryX, sort, "IGAEC");
- assertMatches(full, queryY, sort, "DHFJB");
-
- sort.setSort(new SortField("float", SortField.Type.FLOAT), SortField.FIELD_DOC);
- assertMatches(full, queryX, sort, "GCIEA");
- assertMatches(full, queryY, sort, "DHJFB");
-
- sort.setSort(new SortField("long", SortField.Type.LONG), SortField.FIELD_DOC);
- assertMatches(full, queryX, sort, "EACGI");
- assertMatches(full, queryY, sort, "FBJHD");
-
- sort.setSort(new SortField("double", SortField.Type.DOUBLE), SortField.FIELD_DOC);
- assertMatches(full, queryX, sort, "AGICE");
- assertMatches(full, queryY, sort, "DJHBF");
-
- sort.setSort(new SortField("byte", SortField.Type.BYTE), SortField.FIELD_DOC);
- assertMatches(full, queryX, sort, "CIGAE");
- assertMatches(full, queryY, sort, "DHFBJ");
-
- sort.setSort(new SortField("short", SortField.Type.SHORT), SortField.FIELD_DOC);
- assertMatches(full, queryX, sort, "IAGCE");
- assertMatches(full, queryY, sort, "DFHBJ");
-
- sort.setSort(new SortField("string", SortField.Type.STRING), SortField.FIELD_DOC);
- assertMatches(full, queryX, sort, "AIGEC");
- assertMatches(full, queryY, sort, "DJHFB");
-
- sort.setSort(new SortField("int_dv", SortField.Type.INT), SortField.FIELD_DOC);
- assertMatches(full, queryX, sort, "IGAEC");
- assertMatches(full, queryY, sort, "DHFJB");
-
- sort.setSort(new SortField("float_dv", SortField.Type.FLOAT), SortField.FIELD_DOC);
- assertMatches(full, queryX, sort, "GCIEA");
- assertMatches(full, queryY, sort, "DHJFB");
-
- sort.setSort(new SortField("double_dv", SortField.Type.DOUBLE), SortField.FIELD_DOC);
- assertMatches(full, queryX, sort, "AGICE");
- assertMatches(full, queryY, sort, "DJHBF");
-
- sort.setSort(new SortField("string_dv", getDVStringSortType()), SortField.FIELD_DOC);
- assertMatches(full, queryX, sort, "AIGEC");
- assertMatches(full, queryY, sort, "DJHFB");
- }
-
private SortField.Type getDVStringSortType() {
return getDVStringSortType(true);
}
@@ -609,142 +537,6 @@ public class TestSort extends LuceneTest
assertFalse("Found sort results out of order", fail);
searcher.getIndexReader().close();
}
-
- /**
- * test sorts where the type of field is specified and a custom field parser
- * is used, that uses a simple char encoding. The sorted string contains a
- * character beginning from 'A' that is mapped to a numeric value using some
- * "funny" algorithm to be different for each data type.
- */
- public void testCustomFieldParserSort() throws Exception {
- // since tests explicilty uses different parsers on the same fieldname
- // we explicitly check/purge the FieldCache between each assertMatch
- FieldCache fc = FieldCache.DEFAULT;
-
-
- sort.setSort(new SortField("parser", new FieldCache.IntParser(){
- @Override
- public final int parseInt(final BytesRef term) {
- return (term.bytes[term.offset]-'A') * 123456;
- }
-
- @Override
- public TermsEnum termsEnum(Terms terms) throws IOException {
- return terms.iterator(null);
- }
- }), SortField.FIELD_DOC );
- assertMatches (full, queryA, sort, "JIHGFEDCBA");
- assertSaneFieldCaches(getTestName() + " IntParser");
- fc.purgeAllCaches();
-
- sort.setSort(new SortField("parser", new FieldCache.FloatParser(){
- @Override
- public final float parseFloat(final BytesRef term) {
- return (float) Math.sqrt( term.bytes[term.offset]);
- }
- @Override
- public TermsEnum termsEnum(Terms terms) throws IOException {
- return terms.iterator(null);
- }
- }), SortField.FIELD_DOC );
- assertMatches (full, queryA, sort, "JIHGFEDCBA");
- assertSaneFieldCaches(getTestName() + " FloatParser");
- fc.purgeAllCaches();
-
- sort.setSort(new SortField("parser", new FieldCache.LongParser(){
- @Override
- public final long parseLong(final BytesRef term) {
- return (term.bytes[term.offset]-'A') * 1234567890L;
- }
-
- @Override
- public TermsEnum termsEnum(Terms terms) throws IOException {
- return terms.iterator(null);
- }
- }), SortField.FIELD_DOC );
- assertMatches (full, queryA, sort, "JIHGFEDCBA");
- assertSaneFieldCaches(getTestName() + " LongParser");
- fc.purgeAllCaches();
-
- sort.setSort(new SortField("parser", new FieldCache.DoubleParser(){
- @Override
- public final double parseDouble(final BytesRef term) {
- return Math.pow( term.bytes[term.offset], (term.bytes[term.offset]-'A'));
- }
- @Override
- public TermsEnum termsEnum(Terms terms) throws IOException {
- return terms.iterator(null);
- }
- }), SortField.FIELD_DOC );
- assertMatches (full, queryA, sort, "JIHGFEDCBA");
- assertSaneFieldCaches(getTestName() + " DoubleParser");
- fc.purgeAllCaches();
-
- sort.setSort(new SortField("parser", new FieldCache.ByteParser(){
- @Override
- public final byte parseByte(final BytesRef term) {
- return (byte) (term.bytes[term.offset]-'A');
- }
-
- @Override
- public TermsEnum termsEnum(Terms terms) throws IOException {
- return terms.iterator(null);
- }
- }), SortField.FIELD_DOC );
- assertMatches (full, queryA, sort, "JIHGFEDCBA");
- assertSaneFieldCaches(getTestName() + " ByteParser");
- fc.purgeAllCaches();
-
- sort.setSort(new SortField("parser", new FieldCache.ShortParser(){
- @Override
- public final short parseShort(final BytesRef term) {
- return (short) (term.bytes[term.offset]-'A');
- }
- @Override
- public TermsEnum termsEnum(Terms terms) throws IOException {
- return terms.iterator(null);
- }
- }), SortField.FIELD_DOC );
- assertMatches (full, queryA, sort, "JIHGFEDCBA");
- assertSaneFieldCaches(getTestName() + " ShortParser");
- fc.purgeAllCaches();
- }
-
- // test sorts when there's nothing in the index
- public void testEmptyIndex() throws Exception {
- IndexSearcher empty = getEmptyIndex();
-
- sort = new Sort();
- assertMatches(empty, queryX, sort, "");
-
- sort.setSort(SortField.FIELD_DOC);
- assertMatches(empty, queryX, sort, "");
-
- sort.setSort(new SortField("int", SortField.Type.INT), SortField.FIELD_DOC);
- assertMatches(empty, queryX, sort, "");
-
- sort.setSort(new SortField("int_dv", SortField.Type.INT), SortField.FIELD_DOC);
- assertMatches(empty, queryX, sort, "");
-
- sort.setSort(new SortField("string", SortField.Type.STRING, true), SortField.FIELD_DOC);
- assertMatches(empty, queryX, sort, "");
-
- sort.setSort(new SortField("float", SortField.Type.FLOAT), new SortField("string", SortField.Type.STRING));
- assertMatches(empty, queryX, sort, "");
-
- sort.setSort(new SortField("float_dv", SortField.Type.FLOAT), new SortField("string", SortField.Type.STRING));
- assertMatches(empty, queryX, sort, "");
-
- sort.setSort(new SortField("string_dv", getDVStringSortType(false), true), SortField.FIELD_DOC);
- assertMatches(empty, queryX, sort, "");
-
- sort.setSort(new SortField("float_dv", SortField.Type.FLOAT),
- new SortField("string_dv", getDVStringSortType(false)));
- assertMatches(empty, queryX, sort, "");
-
- sort.setSort(new SortField("float_dv", SortField.Type.FLOAT), new SortField("string_dv", getDVStringSortType(false)));
- assertMatches(empty, queryX, sort, "");
- }
static class MyFieldComparator extends FieldComparator<Integer> {
FieldCache.Ints docValues;
@@ -822,41 +614,6 @@ public class TestSort extends LuceneTest
assertMatches(full, queryA, sort, "JIHGFEDCBA");
}
- // test sorts in reverse
- public void testReverseSort() throws Exception {
- sort.setSort(new SortField(null, SortField.Type.SCORE, true), SortField.FIELD_DOC);
- assertMatches(full, queryX, sort, "IEGCA");
- assertMatches(full, queryY, sort, "JFHDB");
-
- sort.setSort(new SortField(null, SortField.Type.DOC, true));
- assertMatches(full, queryX, sort, "IGECA");
- assertMatches(full, queryY, sort, "JHFDB");
-
- sort.setSort(new SortField("int", SortField.Type.INT, true));
- assertMatches(full, queryX, sort, "CAEGI");
- assertMatches(full, queryY, sort, "BJFHD");
-
- sort.setSort(new SortField("float", SortField.Type.FLOAT, true));
- assertMatches(full, queryX, sort, "AECIG");
- assertMatches(full, queryY, sort, "BFJHD");
-
- sort.setSort(new SortField("string", SortField.Type.STRING, true));
- assertMatches(full, queryX, sort, "CEGIA");
- assertMatches(full, queryY, sort, "BFHJD");
-
- sort.setSort(new SortField("int_dv", SortField.Type.INT, true));
- assertMatches(full, queryX, sort, "CAEGI");
- assertMatches(full, queryY, sort, "BJFHD");
-
- sort.setSort(new SortField("float_dv", SortField.Type.FLOAT, true));
- assertMatches(full, queryX, sort, "AECIG");
- assertMatches(full, queryY, sort, "BFJHD");
-
- sort.setSort(new SortField("string_dv", getDVStringSortType(), true));
- assertMatches(full, queryX, sort, "CEGIA");
- assertMatches(full, queryY, sort, "BFHJD");
- }
-
// test sorting when the sort field is empty (undefined) for some of the documents
public void testEmptyFieldSort() throws Exception {
@@ -980,219 +737,6 @@ public class TestSort extends LuceneTest
assertEquals(docs1.scoreDocs[0].score, docs2.scoreDocs[0].score, 1e-6);
}
- public void testSortWithoutFillFields() throws Exception {
-
- // There was previously a bug in TopFieldCollector when fillFields was set
- // to false - the same doc and score was set in ScoreDoc[] array. This test
- // asserts that if fillFields is false, the documents are set properly. It
- // does not use Searcher's default search methods (with Sort) since all set
- // fillFields to true.
- Sort[] sort = new Sort[] { new Sort(SortField.FIELD_DOC), new Sort() };
- for(int i = 0; i < sort.length; i++) {
- Query q = new MatchAllDocsQuery();
- TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, false,
- false, false, true);
-
- full.search(q, tdc);
-
- ScoreDoc[] sd = tdc.topDocs().scoreDocs;
- for(int j = 1; j < sd.length; j++) {
- assertTrue(sd[j].doc != sd[j - 1].doc);
- }
-
- }
- }
-
- public void testSortWithoutScoreTracking() throws Exception {
-
- // Two Sort criteria to instantiate the multi/single comparators.
- Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
- for(int i = 0; i < sort.length; i++) {
- Query q = new MatchAllDocsQuery();
- TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, false,
- false, true);
-
- full.search(q, tdc);
-
- TopDocs td = tdc.topDocs();
- ScoreDoc[] sd = td.scoreDocs;
- for(int j = 0; j < sd.length; j++) {
- assertTrue(Float.isNaN(sd[j].score));
- }
- assertTrue(Float.isNaN(td.getMaxScore()));
- }
- }
-
- public void testSortWithScoreNoMaxScoreTracking() throws Exception {
-
- // Two Sort criteria to instantiate the multi/single comparators.
- Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
- for(int i = 0; i < sort.length; i++) {
- Query q = new MatchAllDocsQuery();
- TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true,
- false, true);
-
- full.search(q, tdc);
-
- TopDocs td = tdc.topDocs();
- ScoreDoc[] sd = td.scoreDocs;
- for(int j = 0; j < sd.length; j++) {
- assertTrue(!Float.isNaN(sd[j].score));
- }
- assertTrue(Float.isNaN(td.getMaxScore()));
- }
- }
-
- // MultiComparatorScoringNoMaxScoreCollector
- public void testSortWithScoreNoMaxScoreTrackingMulti() throws Exception {
-
- // Two Sort criteria to instantiate the multi/single comparators.
- Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC, SortField.FIELD_SCORE) };
- for(int i = 0; i < sort.length; i++) {
- Query q = new MatchAllDocsQuery();
- TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true,
- false, true);
-
- full.search(q, tdc);
-
- TopDocs td = tdc.topDocs();
- ScoreDoc[] sd = td.scoreDocs;
- for(int j = 0; j < sd.length; j++) {
- assertTrue(!Float.isNaN(sd[j].score));
- }
- assertTrue(Float.isNaN(td.getMaxScore()));
- }
- }
-
- public void testSortWithScoreAndMaxScoreTracking() throws Exception {
-
- // Two Sort criteria to instantiate the multi/single comparators.
- Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
- for(int i = 0; i < sort.length; i++) {
- Query q = new MatchAllDocsQuery();
- TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true,
- true, true);
-
- full.search(q, tdc);
-
- TopDocs td = tdc.topDocs();
- ScoreDoc[] sd = td.scoreDocs;
- for(int j = 0; j < sd.length; j++) {
- assertTrue(!Float.isNaN(sd[j].score));
- }
- assertTrue(!Float.isNaN(td.getMaxScore()));
- }
- }
-
- public void testOutOfOrderDocsScoringSort() throws Exception {
-
- // Two Sort criteria to instantiate the multi/single comparators.
- Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
- boolean[][] tfcOptions = new boolean[][] {
- new boolean[] { false, false, false },
- new boolean[] { false, false, true },
- new boolean[] { false, true, false },
- new boolean[] { false, true, true },
- new boolean[] { true, false, false },
- new boolean[] { true, false, true },
- new boolean[] { true, true, false },
- new boolean[] { true, true, true },
- };
- String[] actualTFCClasses = new String[] {
- "OutOfOrderOneComparatorNonScoringCollector",
- "OutOfOrderOneComparatorScoringMaxScoreCollector",
- "OutOfOrderOneComparatorScoringNoMaxScoreCollector",
- "OutOfOrderOneComparatorScoringMaxScoreCollector",
- "OutOfOrderOneComparatorNonScoringCollector",
- "OutOfOrderOneComparatorScoringMaxScoreCollector",
- "OutOfOrderOneComparatorScoringNoMaxScoreCollector",
- "OutOfOrderOneComparatorScoringMaxScoreCollector"
- };
-
- BooleanQuery bq = new BooleanQuery();
- // Add a Query with SHOULD, since bw.scorer() returns BooleanScorer2
- // which delegates to BS if there are no mandatory clauses.
- bq.add(new MatchAllDocsQuery(), Occur.SHOULD);
- // Set minNrShouldMatch to 1 so that BQ will not optimize rewrite to return
- // the clause instead of BQ.
- bq.setMinimumNumberShouldMatch(1);
- for(int i = 0; i < sort.length; i++) {
- for(int j = 0; j < tfcOptions.length; j++) {
- TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10,
- tfcOptions[j][0], tfcOptions[j][1], tfcOptions[j][2], false);
-
- assertTrue(tdc.getClass().getName().endsWith("$"+actualTFCClasses[j]));
-
- full.search(bq, tdc);
-
- TopDocs td = tdc.topDocs();
- ScoreDoc[] sd = td.scoreDocs;
- assertEquals(10, sd.length);
- }
- }
- }
-
- // OutOfOrderMulti*Collector
- public void testOutOfOrderDocsScoringSortMulti() throws Exception {
-
- // Two Sort criteria to instantiate the multi/single comparators.
- Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC, SortField.FIELD_SCORE) };
- boolean[][] tfcOptions = new boolean[][] {
- new boolean[] { false, false, false },
- new boolean[] { false, false, true },
- new boolean[] { false, true, false },
- new boolean[] { false, true, true },
- new boolean[] { true, false, false },
- new boolean[] { true, false, true },
- new boolean[] { true, true, false },
- new boolean[] { true, true, true },
- };
- String[] actualTFCClasses = new String[] {
- "OutOfOrderMultiComparatorNonScoringCollector",
- "OutOfOrderMultiComparatorScoringMaxScoreCollector",
- "OutOfOrderMultiComparatorScoringNoMaxScoreCollector",
- "OutOfOrderMultiComparatorScoringMaxScoreCollector",
- "OutOfOrderMultiComparatorNonScoringCollector",
- "OutOfOrderMultiComparatorScoringMaxScoreCollector",
- "OutOfOrderMultiComparatorScoringNoMaxScoreCollector",
- "OutOfOrderMultiComparatorScoringMaxScoreCollector"
- };
-
- BooleanQuery bq = new BooleanQuery();
- // Add a Query with SHOULD, since bw.scorer() returns BooleanScorer2
- // which delegates to BS if there are no mandatory clauses.
- bq.add(new MatchAllDocsQuery(), Occur.SHOULD);
- // Set minNrShouldMatch to 1 so that BQ will not optimize rewrite to return
- // the clause instead of BQ.
- bq.setMinimumNumberShouldMatch(1);
- for(int i = 0; i < sort.length; i++) {
- for(int j = 0; j < tfcOptions.length; j++) {
- TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10,
- tfcOptions[j][0], tfcOptions[j][1], tfcOptions[j][2], false);
-
- assertTrue(tdc.getClass().getName().endsWith("$"+actualTFCClasses[j]));
-
- full.search(bq, tdc);
-
- TopDocs td = tdc.topDocs();
- ScoreDoc[] sd = td.scoreDocs;
- assertEquals(10, sd.length);
- }
- }
- }
-
- public void testSortWithScoreAndMaxScoreTrackingNoResults() throws Exception {
-
- // Two Sort criteria to instantiate the multi/single comparators.
- Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
- for(int i = 0; i < sort.length; i++) {
- TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true, true, true);
- TopDocs td = tdc.topDocs();
- assertEquals(0, td.totalHits);
- assertTrue(Float.isNaN(td.getMaxScore()));
- }
- }
-
// runs a variety of sorts useful for multisearchers
private void runMultiSorts(IndexSearcher multi, boolean isFull) throws Exception {
sort.setSort(SortField.FIELD_DOC);
@@ -1331,255 +875,4 @@ public class TestSort extends LuceneTest
}
assertEquals(msg, expectedResult, buff.toString());
}
-
- public void testEmptyStringVsNullStringSort() throws Exception {
- Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random())));
- Document doc = new Document();
- doc.add(newStringField("f", "", Field.Store.NO));
- doc.add(newStringField("t", "1", Field.Store.NO));
- w.addDocument(doc);
- w.commit();
- doc = new Document();
- doc.add(newStringField("t", "1", Field.Store.NO));
- w.addDocument(doc);
-
- IndexReader r = DirectoryReader.open(w, true);
- w.close();
- IndexSearcher s = newSearcher(r);
- TopDocs hits = s.search(new TermQuery(new Term("t", "1")), null, 10, new Sort(new SortField("f", SortField.Type.STRING)));
- assertEquals(2, hits.totalHits);
- // null sorts first
- assertEquals(1, hits.scoreDocs[0].doc);
- assertEquals(0, hits.scoreDocs[1].doc);
- r.close();
- dir.close();
- }
-
- public void testLUCENE2142() throws IOException {
- Directory indexStore = newDirectory();
- IndexWriter writer = new IndexWriter(indexStore, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random())));
- for(int i=0; i<5; i++) {
- Document doc = new Document();
- doc.add(new StringField("string", "a"+i, Field.Store.NO));
- doc.add(new StringField("string", "b"+i, Field.Store.NO));
- writer.addDocument(doc);
- }
- writer.forceMerge(1); // enforce one segment to have a higher unique term count in all cases
- writer.close();
- sort.setSort(
- new SortField("string", SortField.Type.STRING),
- SortField.FIELD_DOC);
- // this should not throw AIOOBE or RuntimeEx
- IndexReader reader = DirectoryReader.open(indexStore);
- IndexSearcher searcher = new IndexSearcher(reader);
- searcher.search(new MatchAllDocsQuery(), null, 500, sort);
- reader.close();
- indexStore.close();
- }
-
- public void testCountingCollector() throws Exception {
- Directory indexStore = newDirectory();
- RandomIndexWriter writer = new RandomIndexWriter(random(), indexStore);
- for(int i=0; i<5; i++) {
- Document doc = new Document();
- doc.add(new StringField("string", "a"+i, Field.Store.NO));
- doc.add(new StringField("string", "b"+i, Field.Store.NO));
- writer.addDocument(doc);
- }
- IndexReader reader = writer.getReader();
- writer.close();
-
- IndexSearcher searcher = newSearcher(reader);
- TotalHitCountCollector c = new TotalHitCountCollector();
- searcher.search(new MatchAllDocsQuery(), null, c);
- assertEquals(5, c.getTotalHits());
- reader.close();
- indexStore.close();
- }
-
- private static class RandomFilter extends Filter {
- private final Random random;
- private float density;
- private final List<BytesRef> docValues;
- public final List<BytesRef> matchValues = Collections.synchronizedList(new ArrayList<BytesRef>());
-
- // density should be 0.0 ... 1.0
- public RandomFilter(Random random, float density, List<BytesRef> docValues) {
- this.random = random;
- this.density = density;
- this.docValues = docValues;
- }
-
- @Override
- public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
- final int maxDoc = context.reader().maxDoc();
- final FieldCache.Ints idSource = FieldCache.DEFAULT.getInts(context.reader(), "id", false);
- assertNotNull(idSource);
- final FixedBitSet bits = new FixedBitSet(maxDoc);
- for(int docID=0;docID<maxDoc;docID++) {
- if (random.nextFloat() <= density && (acceptDocs == null || acceptDocs.get(docID))) {
- bits.set(docID);
- //System.out.println(" acc id=" + idSource.getInt(docID) + " docID=" + docID);
- matchValues.add(docValues.get(idSource.get(docID)));
- }
- }
-
- return bits;
- }
- }
-
- public void testRandomStringSort() throws Exception {
- Random random = new Random(random().nextLong());
-
- final int NUM_DOCS = atLeast(100);
- final Directory dir = newDirectory();
- final RandomIndexWriter writer = new RandomIndexWriter(random, dir);
- final boolean allowDups = random.nextBoolean();
- final Set<String> seen = new HashSet<String>();
- final int maxLength = _TestUtil.nextInt(random, 5, 100);
- if (VERBOSE) {
- System.out.println("TEST: NUM_DOCS=" + NUM_DOCS + " maxLength=" + maxLength + " allowDups=" + allowDups);
- }
-
- int numDocs = 0;
- final List<BytesRef> docValues = new ArrayList<BytesRef>();
- // TODO: deletions
- while (numDocs < NUM_DOCS) {
- final String s;
- if (random.nextBoolean()) {
- s = _TestUtil.randomSimpleString(random, maxLength);
- } else {
- s = _TestUtil.randomUnicodeString(random, maxLength);
- }
- final BytesRef br = new BytesRef(s);
-
- if (!allowDups) {
- if (seen.contains(s)) {
- continue;
- }
- seen.add(s);
- }
-
- if (VERBOSE) {
- System.out.println(" " + numDocs + ": s=" + s);
- }
-
- final Document doc = new Document();
- doc.add(new SortedDocValuesField("stringdv", br));
- doc.add(newStringField("string", s, Field.Store.NO));
- doc.add(new NumericDocValuesField("id", numDocs));
- docValues.add(br);
- writer.addDocument(doc);
- numDocs++;
-
- if (random.nextInt(40) == 17) {
- // force flush
- writer.getReader().close();
- }
- }
-
- final IndexReader r = writer.getReader();
- writer.close();
- if (VERBOSE) {
- System.out.println(" reader=" + r);
- }
-
- final IndexSearcher s = newSearcher(r, false);
- final int ITERS = atLeast(100);
- for(int iter=0;iter<ITERS;iter++) {
- final boolean reverse = random.nextBoolean();
- final TopFieldDocs hits;
- final SortField sf;
- if (random.nextBoolean()) {
- sf = new SortField("stringdv", SortField.Type.STRING, reverse);
- } else {
- sf = new SortField("string", SortField.Type.STRING, reverse);
- }
- final Sort sort = new Sort(sf);
- final int hitCount = _TestUtil.nextInt(random, 1, r.maxDoc() + 20);
- final RandomFilter f = new RandomFilter(random, random.nextFloat(), docValues);
- if (random.nextBoolean()) {
- hits = s.search(new ConstantScoreQuery(f),
- hitCount,
- sort);
- } else {
- hits = s.search(new MatchAllDocsQuery(),
- f,
- hitCount,
- sort);
- }
-
- if (VERBOSE) {
- System.out.println("\nTEST: iter=" + iter + " " + hits.totalHits + " hits; topN=" + hitCount + "; reverse=" + reverse);
- }
-
- // Compute expected results:
- Collections.sort(f.matchValues);
- if (reverse) {
- Collections.reverse(f.matchValues);
- }
- final List<BytesRef> expected = f.matchValues;
- if (VERBOSE) {
- System.out.println(" expected:");
- for(int idx=0;idx<expected.size();idx++) {
- System.out.println(" " + idx + ": " + expected.get(idx).utf8ToString());
- if (idx == hitCount-1) {
- break;
- }
- }
- }
-
- if (VERBOSE) {
- System.out.println(" actual:");
- for(int hitIDX=0;hitIDX<hits.scoreDocs.length;hitIDX++) {
- final FieldDoc fd = (FieldDoc) hits.scoreDocs[hitIDX];
- System.out.println(" " + hitIDX + ": " + ((BytesRef) fd.fields[0]).utf8ToString());
- }
- }
- for(int hitIDX=0;hitIDX<hits.scoreDocs.length;hitIDX++) {
- final FieldDoc fd = (FieldDoc) hits.scoreDocs[hitIDX];
- assertEquals(expected.get(hitIDX), (BytesRef) fd.fields[0]);
- }
- }
-
- r.close();
- dir.close();
- }
-
- public void testMaxScore() throws Exception {
- Directory d = newDirectory();
- // Not RIW because we need exactly 2 segs:
- IndexWriter w = new IndexWriter(d, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
- int id = 0;
- for(int seg=0;seg<2;seg++) {
- for(int docIDX=0;docIDX<10;docIDX++) {
- Document doc = new Document();
- doc.add(newStringField("id", ""+docIDX, Field.Store.YES));
- StringBuilder sb = new StringBuilder();
- for(int i=0;i<id;i++) {
- sb.append(' ');
- sb.append("text");
- }
- doc.add(newTextField("body", sb.toString(), Field.Store.NO));
- w.addDocument(doc);
- id++;
- }
- w.commit();
- }
-
- IndexReader r = DirectoryReader.open(w, true);
- w.close();
- Query q = new TermQuery(new Term("body", "text"));
- IndexSearcher s = newSearcher(r);
- float maxScore = s.search(q , 10).getMaxScore();
- assertEquals(maxScore, s.search(q, null, 3, Sort.INDEXORDER, random().nextBoolean(), true).getMaxScore(), 0.0);
- assertEquals(maxScore, s.search(q, null, 3, Sort.RELEVANCE, random().nextBoolean(), true).getMaxScore(), 0.0);
- assertEquals(maxScore, s.search(q, null, 3, new Sort(new SortField[] {new SortField("id", SortField.Type.INT, false)}), random().nextBoolean(), true).getMaxScore(), 0.0);
- assertEquals(maxScore, s.search(q, null, 3, new Sort(new SortField[] {new SortField("id", SortField.Type.INT, true)}), random().nextBoolean(), true).getMaxScore(), 0.0);
- r.close();
- d.close();
- }
}
Modified: lucene/dev/branches/lucene4765/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java (original)
+++ lucene/dev/branches/lucene4765/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java Fri Feb 15 22:58:27 2013
@@ -145,7 +145,9 @@ public class TestDocValuesFieldSources e
public void test() throws IOException {
for (DocValuesType type : DocValuesType.values()) {
- test(type);
+ if (type != DocValuesType.SORTED_SET) {
+ test(type);
+ }
}
}
Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java Fri Feb 15 22:58:27 2013
@@ -78,12 +78,14 @@ import java.util.concurrent.ConcurrentHa
public abstract class PrefixTreeStrategy extends SpatialStrategy {
protected final SpatialPrefixTree grid;
private final Map<String, PointPrefixTreeFieldCacheProvider> provider = new ConcurrentHashMap<String, PointPrefixTreeFieldCacheProvider>();
+ protected final boolean simplifyIndexedCells;
protected int defaultFieldValuesArrayLen = 2;
protected double distErrPct = SpatialArgs.DEFAULT_DISTERRPCT;// [ 0 TO 0.5 ]
- public PrefixTreeStrategy(SpatialPrefixTree grid, String fieldName) {
+ public PrefixTreeStrategy(SpatialPrefixTree grid, String fieldName, boolean simplifyIndexedCells) {
super(grid.getSpatialContext(), fieldName);
this.grid = grid;
+ this.simplifyIndexedCells = simplifyIndexedCells;
}
/**
@@ -123,7 +125,7 @@ public abstract class PrefixTreeStrategy
public Field[] createIndexableFields(Shape shape, double distErr) {
int detailLevel = grid.getLevelForDistance(distErr);
- List<Node> cells = grid.getNodes(shape, detailLevel, true);//true=intermediates cells
+ List<Node> cells = grid.getNodes(shape, detailLevel, true, simplifyIndexedCells);//intermediates cells
//TODO is CellTokenStream supposed to be re-used somehow? see Uwe's comments:
// http://code.google.com/p/lucene-spatial-playground/issues/detail?id=4
Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java Fri Feb 15 22:58:27 2013
@@ -38,7 +38,8 @@ public class RecursivePrefixTreeStrategy
private int prefixGridScanLevel;
public RecursivePrefixTreeStrategy(SpatialPrefixTree grid, String fieldName) {
- super(grid, fieldName);
+ super(grid, fieldName,
+ true);//simplify indexed cells
prefixGridScanLevel = grid.getMaxLevels() - 4;//TODO this default constant is dependent on the prefix grid size
}
Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java Fri Feb 15 22:58:27 2013
@@ -32,7 +32,7 @@ import java.util.List;
/**
* A basic implementation of {@link PrefixTreeStrategy} using a large {@link
* TermsFilter} of all the nodes from {@link SpatialPrefixTree#getNodes(com.spatial4j.core.shape.Shape,
- * int, boolean)}. It only supports the search of indexed Point shapes.
+ * int, boolean, boolean)}. It only supports the search of indexed Point shapes.
* <p/>
* The precision of query shapes (distErrPct) is an important factor in using
* this Strategy. If the precision is too precise then it will result in many
@@ -43,7 +43,8 @@ import java.util.List;
public class TermQueryPrefixTreeStrategy extends PrefixTreeStrategy {
public TermQueryPrefixTreeStrategy(SpatialPrefixTree grid, String fieldName) {
- super(grid, fieldName);
+ super(grid, fieldName,
+ false);//do not simplify indexed cells
}
@Override
@@ -54,7 +55,9 @@ public class TermQueryPrefixTreeStrategy
Shape shape = args.getShape();
int detailLevel = grid.getLevelForDistance(args.resolveDistErr(ctx, distErrPct));
- List<Node> cells = grid.getNodes(shape, detailLevel, false);
+ List<Node> cells = grid.getNodes(shape, detailLevel,
+ false,//no parents
+ true);//simplify
BytesRef[] terms = new BytesRef[cells.size()];
int i = 0;
for (Node cell : cells) {
Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java Fri Feb 15 22:58:27 2013
@@ -93,12 +93,6 @@ public class GeohashPrefixTree extends S
return new GhCell(bytes, offset, len);
}
- @Override
- public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents) {
- return shape instanceof Point ? super.getNodesAltPoint((Point) shape, detailLevel, inclParents) :
- super.getNodes(shape, detailLevel, inclParents);
- }
-
class GhCell extends Node {
GhCell(String token) {
super(token);
Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java Fri Feb 15 22:58:27 2013
@@ -156,14 +156,6 @@ public class QuadPrefixTree extends Spat
return new QuadCell(bytes, offset, len);
}
- @Override //for performance
- public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents) {
- if (shape instanceof Point)
- return super.getNodesAltPoint((Point) shape, detailLevel, inclParents);
- else
- return super.getNodes(shape, detailLevel, inclParents);
- }
-
private void build(
double x,
double y,
Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java Fri Feb 15 22:58:27 2013
@@ -116,88 +116,86 @@ public abstract class SpatialPrefixTree
}
/**
- * Gets the intersecting & including cells for the specified shape, without exceeding detail level.
- * The result is a set of cells (no dups), sorted. Unmodifiable.
+ * Gets the intersecting cells for the specified shape, without exceeding
+ * detail level. If a cell is within the query shape then it's marked as a
+ * leaf and none of its children are added.
* <p/>
- * This implementation checks if shape is a Point and if so uses an implementation that
- * recursively calls {@link Node#getSubCell(com.spatial4j.core.shape.Point)}. Cell subclasses
- * ideally implement that method with a quick implementation, otherwise, subclasses should
- * override this method to invoke {@link #getNodesAltPoint(com.spatial4j.core.shape.Point, int, boolean)}.
- * TODO consider another approach returning an iterator -- won't build up all cells in memory.
+ * This implementation checks if shape is a Point and if so returns {@link
+ * #getNodes(com.spatial4j.core.shape.Point, int, boolean)}.
+ *
+ * @param shape the shape; non-null
+ * @param detailLevel the maximum detail level to get cells for
+ * @param inclParents if true then all parent cells of leaves are returned
+ * too. The top world cell is never returned.
+ * @param simplify for non-point shapes, this will simply/aggregate sets of
+ * complete leaves in a cell to its parent, resulting in
+ * ~20-25% fewer cells.
+ * @return a set of cells (no dups), sorted, immutable, non-null
*/
- public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents) {
+ public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents,
+ boolean simplify) {
+ //TODO consider an on-demand iterator -- it won't build up all cells in memory.
if (detailLevel > maxLevels) {
throw new IllegalArgumentException("detailLevel > maxLevels");
}
-
- List<Node> cells;
if (shape instanceof Point) {
- //optimized point algorithm
- final int initialCapacity = inclParents ? 1 + detailLevel : 1;
- cells = new ArrayList<Node>(initialCapacity);
- recursiveGetNodes(getWorldNode(), (Point) shape, detailLevel, true, cells);
- assert cells.size() == initialCapacity;
- } else {
- cells = new ArrayList<Node>(inclParents ? 1024 : 512);
- recursiveGetNodes(getWorldNode(), shape, detailLevel, inclParents, cells);
- }
- if (inclParents) {
- Node c = cells.remove(0);//remove getWorldNode()
- assert c.getLevel() == 0;
+ return getNodes((Point) shape, detailLevel, inclParents);
}
+ List<Node> cells = new ArrayList<Node>(inclParents ? 4096 : 2048);
+ recursiveGetNodes(getWorldNode(), shape, detailLevel, inclParents, simplify, cells);
return cells;
}
- private void recursiveGetNodes(Node node, Shape shape, int detailLevel, boolean inclParents,
- Collection<Node> result) {
- if (node.isLeaf()) {//cell is within shape
- result.add(node);
- return;
- }
- final Collection<Node> subCells = node.getSubCells(shape);
- if (node.getLevel() == detailLevel - 1) {
- if (subCells.size() < node.getSubCellsSize() || node.getLevel() == 0) {
- if (inclParents)
- result.add(node);
- for (Node subCell : subCells) {
- subCell.setLeaf();
- }
- result.addAll(subCells);
- } else {//a bottom level (i.e. detail level) optimization where all boxes intersect, so use parent cell.
- node.setLeaf();//the cell may not be strictly within but its close
- result.add(node);
- }
- } else {
- if (inclParents) {
- result.add(node);
- }
- for (Node subCell : subCells) {
- recursiveGetNodes(subCell, shape, detailLevel, inclParents, result);//tail call
- }
+ /**
+ * Returns true if node was added as a leaf. If it wasn't it recursively
+ * descends.
+ */
+ private boolean recursiveGetNodes(Node node, Shape shape, int detailLevel,
+ boolean inclParents, boolean simplify,
+ List<Node> result) {
+ if (node.getLevel() == detailLevel) {
+ node.setLeaf();//FYI might already be a leaf
}
- }
-
- private void recursiveGetNodes(Node node, Point point, int detailLevel, boolean inclParents,
- Collection<Node> result) {
- if (inclParents) {
+ if (node.isLeaf()) {
result.add(node);
+ return true;
}
- final Node pCell = node.getSubCell(point);
- if (node.getLevel() == detailLevel - 1) {
- pCell.setLeaf();
- result.add(pCell);
- } else {
- recursiveGetNodes(pCell, point, detailLevel, inclParents, result);//tail call
+ if (inclParents && node.getLevel() != 0)
+ result.add(node);
+
+ Collection<Node> subCells = node.getSubCells(shape);
+ int leaves = 0;
+ for (Node subCell : subCells) {
+ if (recursiveGetNodes(subCell, shape, detailLevel, inclParents, simplify, result))
+ leaves++;
+ }
+ //can we simplify?
+ if (simplify && leaves == node.getSubCellsSize() && node.getLevel() != 0) {
+ //Optimization: substitute the parent as a leaf instead of adding all
+ // children as leaves
+
+ //remove the leaves
+ do {
+ result.remove(result.size() - 1);//remove last
+ } while (--leaves > 0);
+ //add node as the leaf
+ node.setLeaf();
+ if (!inclParents) // otherwise it was already added up above
+ result.add(node);
+ return true;
}
+ return false;
}
/**
- * Subclasses might override {@link #getNodes(com.spatial4j.core.shape.Shape, int, boolean)}
- * and check if the argument is a shape and if so, delegate
- * to this implementation, which calls {@link #getNode(com.spatial4j.core.shape.Point, int)} and
- * then calls {@link #getNode(String)} repeatedly if inclParents is true.
+ * A Point-optimized implementation of
+ * {@link #getNodes(com.spatial4j.core.shape.Shape, int, boolean, boolean)}. That
+ * method in facts calls this for points.
+ * <p/>
+ * This implementation depends on {@link #getNode(String)} being fast, as its
+ * called repeatedly when incPlarents is true.
*/
- protected final List<Node> getNodesAltPoint(Point p, int detailLevel, boolean inclParents) {
+ public List<Node> getNodes(Point p, int detailLevel, boolean inclParents) {
Node cell = getNode(p, detailLevel);
if (!inclParents) {
return Collections.singletonList(cell);
Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java Fri Feb 15 22:58:27 2013
@@ -18,9 +18,19 @@ package org.apache.lucene.spatial.prefix
*/
import com.spatial4j.core.context.SpatialContextFactory;
+import com.spatial4j.core.shape.Point;
import com.spatial4j.core.shape.Shape;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
import org.apache.lucene.spatial.StrategyTestCase;
import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
+import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
+import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.query.SpatialOperation;
import org.junit.Test;
@@ -70,4 +80,39 @@ public class JtsPolygonTest extends Stra
return args;
}
+ /**
+ * A PrefixTree pruning optimization gone bad.
+ * See <a href="https://issues.apache.org/jira/browse/LUCENE-4770>LUCENE-4770</a>.
+ */
+ @Test
+ public void testBadPrefixTreePrune() throws Exception {
+
+ Shape area = ctx.readShape("POLYGON((-122.83 48.57, -122.77 48.56, -122.79 48.53, -122.83 48.57))");
+
+ SpatialPrefixTree trie = new QuadPrefixTree(ctx, 12);
+ TermQueryPrefixTreeStrategy strategy = new TermQueryPrefixTreeStrategy(trie, "geo");
+ Document doc = new Document();
+ doc.add(new TextField("id", "1", Store.YES));
+
+ Field[] fields = strategy.createIndexableFields(area, 0.025);
+ for (Field field : fields) {
+ doc.add(field);
+ }
+ addDocument(doc);
+
+ Point upperleft = ctx.makePoint(-122.88, 48.54);
+ Point lowerright = ctx.makePoint(-122.82, 48.62);
+
+ Query query = strategy.makeQuery(new SpatialArgs(SpatialOperation.Intersects, ctx.makeRectangle(upperleft, lowerright)));
+ commit();
+
+ TopDocs search = indexSearcher.search(query, 10);
+ ScoreDoc[] scoreDocs = search.scoreDocs;
+ for (ScoreDoc scoreDoc : scoreDocs) {
+ System.out.println(indexSearcher.doc(scoreDoc.doc));
+ }
+
+ assertEquals(1, search.totalHits);
+ }
+
}
Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java Fri Feb 15 22:58:27 2013
@@ -113,7 +113,7 @@ public class SpatialOpRecursivePrefixTre
double distErrPct = ((PrefixTreeStrategy) strategy).getDistErrPct();
double distErr = SpatialArgs.calcDistanceFromErrPct(snapMe, distErrPct, ctx);
int detailLevel = grid.getLevelForDistance(distErr);
- List<Node> cells = grid.getNodes(snapMe, detailLevel, false);
+ List<Node> cells = grid.getNodes(snapMe, detailLevel, false, true);
//calc bounding box of cells.
double minX = Double.POSITIVE_INFINITY, maxX = Double.NEGATIVE_INFINITY;
Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeTest.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeTest.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeTest.java Fri Feb 15 22:58:27 2013
@@ -18,14 +18,24 @@
package org.apache.lucene.spatial.prefix.tree;
import com.spatial4j.core.context.SpatialContext;
+import com.spatial4j.core.shape.Point;
import com.spatial4j.core.shape.Rectangle;
import com.spatial4j.core.shape.Shape;
-
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.spatial.SpatialTestCase;
+import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy;
+import org.apache.lucene.spatial.query.SpatialArgs;
+import org.apache.lucene.spatial.query.SpatialOperation;
import org.junit.Before;
import org.junit.Test;
-public class SpatialPrefixTreeTest extends LuceneTestCase {
+public class SpatialPrefixTreeTest extends SpatialTestCase {
//TODO plug in others and test them
private SpatialContext ctx;
@@ -36,11 +46,12 @@ public class SpatialPrefixTreeTest exten
public void setUp() throws Exception {
super.setUp();
ctx = SpatialContext.GEO;
- trie = new GeohashPrefixTree(ctx,4);
}
@Test
public void testNodeTraverse() {
+ trie = new GeohashPrefixTree(ctx,4);
+
Node prevN = null;
Node n = trie.getWorldNode();
assertEquals(0,n.getLevel());
@@ -57,4 +68,40 @@ public class SpatialPrefixTreeTest exten
assertTrue(prevNShape.getHeight() > sbox.getHeight());
}
}
-}
+ /**
+ * A PrefixTree pruning optimization gone bad.
+ * See <a href="https://issues.apache.org/jira/browse/LUCENE-4770>LUCENE-4770</a>.
+ */
+ @Test
+ public void testBadPrefixTreePrune() throws Exception {
+
+ trie = new QuadPrefixTree(ctx, 12);
+ TermQueryPrefixTreeStrategy strategy = new TermQueryPrefixTreeStrategy(trie, "geo");
+ Document doc = new Document();
+ doc.add(new TextField("id", "1", Store.YES));
+
+ Shape area = ctx.makeRectangle(-122.82, -122.78, 48.54, 48.56);
+
+ Field[] fields = strategy.createIndexableFields(area, 0.025);
+ for (Field field : fields) {
+ doc.add(field);
+ }
+ addDocument(doc);
+
+ Point upperleft = ctx.makePoint(-122.88, 48.54);
+ Point lowerright = ctx.makePoint(-122.82, 48.62);
+
+ Query query = strategy.makeQuery(new SpatialArgs(SpatialOperation.Intersects, ctx.makeRectangle(upperleft, lowerright)));
+
+ commit();
+
+ TopDocs search = indexSearcher.search(query, 10);
+ ScoreDoc[] scoreDocs = search.scoreDocs;
+ for (ScoreDoc scoreDoc : scoreDocs) {
+ System.out.println(indexSearcher.doc(scoreDoc.doc));
+ }
+
+ assertEquals(1, search.totalHits);
+ }
+
+}
\ No newline at end of file
Modified: lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java (original)
+++ lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java Fri Feb 15 22:58:27 2013
@@ -80,7 +80,7 @@ public class AssertingTermVectorsFormat
static class AssertingTermVectorsWriter extends TermVectorsWriter {
private final TermVectorsWriter in;
private Status docStatus, fieldStatus, termStatus;
- private int fieldCount, termCount, positionCount;
+ private int docCount, fieldCount, termCount, positionCount;
boolean hasPositions;
AssertingTermVectorsWriter(TermVectorsWriter in) {
@@ -98,6 +98,7 @@ public class AssertingTermVectorsFormat
in.startDocument(numVectorFields);
docStatus = Status.STARTED;
fieldCount = numVectorFields;
+ docCount++;
}
@Override
@@ -167,6 +168,7 @@ public class AssertingTermVectorsFormat
@Override
public void finish(FieldInfos fis, int numDocs) throws IOException {
+ assert docCount == numDocs;
assert docStatus == (numDocs > 0 ? Status.FINISHED : Status.UNDEFINED);
assert fieldStatus != Status.STARTED;
assert termStatus != Status.STARTED;
@@ -181,9 +183,6 @@ public class AssertingTermVectorsFormat
@Override
public void close() throws IOException {
in.close();
- assert docStatus != Status.STARTED;
- assert fieldStatus != Status.STARTED;
- assert termStatus != Status.STARTED;
}
}
Modified: lucene/dev/branches/lucene4765/lucene/tools/junit4/tests.policy
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/tools/junit4/tests.policy?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/tools/junit4/tests.policy (original)
+++ lucene/dev/branches/lucene4765/lucene/tools/junit4/tests.policy Fri Feb 15 22:58:27 2013
@@ -67,4 +67,7 @@ grant {
permission java.security.SecurityPermission "getProperty.networkaddress.cache.ttl";
permission java.security.SecurityPermission "getProperty.networkaddress.cache.negative.ttl";
+ // SSL related properties for Solr tests
+ permission java.security.SecurityPermission "getProperty.ssl.*";
+
};
Modified: lucene/dev/branches/lucene4765/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/CHANGES.txt?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene4765/solr/CHANGES.txt Fri Feb 15 22:58:27 2013
@@ -41,6 +41,8 @@ Detailed Change List
Other Changes
----------------------
+* SOLR-4394: Tests and example configs demonstrating SSL with both server
+ and client certs (hossman)
================== 4.2.0 ==================
@@ -71,6 +73,10 @@ New Features
* SOLR-4370: Allow configuring commitWithin to do hard commits.
(Mark Miller, Senthuran Sivananthan)
+* SOLR-4451: SolrJ, and SolrCloud internals, now use SystemDefaultHttpClient
+ under the covers -- allowing many HTTP connection related properties to be
+ controlled via 'standard' java system properties. (hossman)
+
Bug Fixes
----------------------
@@ -117,6 +123,8 @@ Bug Fixes
* SOLR-4426: NRTCachingDirectoryFactory does not initialize maxCachedMB and maxMergeSizeMB
if <directoryFactory> is not present in solrconfig.xml (Jack Krupansky via shalin)
+* SOLR-4463: Fix SolrCoreState reference counting. (Mark Miller)
+
Optimizations
----------------------
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java Fri Feb 15 22:58:27 2013
@@ -23,6 +23,9 @@ import java.util.LinkedList;
import java.util.Random;
import java.util.concurrent.atomic.AtomicLong;
+import java.net.URL;
+import java.net.MalformedURLException;
+
import javax.servlet.DispatcherType;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
@@ -38,8 +41,11 @@ import org.apache.solr.servlet.SolrDispa
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.bio.SocketConnector;
-import org.eclipse.jetty.server.handler.GzipHandler;
import org.eclipse.jetty.server.nio.SelectChannelConnector;
+import org.eclipse.jetty.server.ssl.SslConnector;
+import org.eclipse.jetty.server.ssl.SslSocketConnector;
+import org.eclipse.jetty.server.ssl.SslSelectChannelConnector;
+import org.eclipse.jetty.server.handler.GzipHandler;
import org.eclipse.jetty.server.session.HashSessionIdManager;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletContextHandler;
@@ -47,6 +53,7 @@ import org.eclipse.jetty.util.component.
import org.eclipse.jetty.util.log.Logger;
import org.eclipse.jetty.util.thread.QueuedThreadPool;
import org.eclipse.jetty.util.thread.ThreadPool;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
/**
* Run solr using jetty
@@ -155,22 +162,59 @@ public class JettySolrRunner {
System.setProperty("solr.solr.home", solrHome);
if (System.getProperty("jetty.testMode") != null) {
final String connectorName = System.getProperty("tests.jettyConnector", "SelectChannel");
+
+ // if this property is true, then jetty will be configured to use SSL
+ // leveraging the same system properties as java to specify
+ // the keystore/truststore if they are set
+ //
+ // This means we will use the same truststore, keystore (and keys) for
+ // the server as well as any client actions taken by this JVM in
+ // talking to that server, but for the purposes of testing that should
+ // be good enough
+ final boolean useSsl = Boolean.getBoolean("tests.jettySsl");
+ final SslContextFactory sslcontext = new SslContextFactory(false);
+
+ if (useSsl) {
+ if (null != System.getProperty("javax.net.ssl.keyStore")) {
+ sslcontext.setKeyStorePath
+ (System.getProperty("javax.net.ssl.keyStore"));
+ }
+ if (null != System.getProperty("javax.net.ssl.keyStorePassword")) {
+ sslcontext.setKeyStorePassword
+ (System.getProperty("javax.net.ssl.keyStorePassword"));
+ }
+ if (null != System.getProperty("javax.net.ssl.trustStore")) {
+ sslcontext.setTrustStore
+ (System.getProperty("javax.net.ssl.trustStore"));
+ }
+ if (null != System.getProperty("javax.net.ssl.trustStorePassword")) {
+ sslcontext.setTrustStorePassword
+ (System.getProperty("javax.net.ssl.trustStorePassword"));
+ }
+ sslcontext.setNeedClientAuth(Boolean.getBoolean("tests.jettySsl.clientAuth"));
+ }
+
final Connector connector;
final QueuedThreadPool threadPool;
if ("SelectChannel".equals(connectorName)) {
- final SelectChannelConnector c = new SelectChannelConnector();
+ final SelectChannelConnector c = useSsl
+ ? new SslSelectChannelConnector(sslcontext)
+ : new SelectChannelConnector();
c.setReuseAddress(true);
c.setLowResourcesMaxIdleTime(1500);
connector = c;
threadPool = (QueuedThreadPool) c.getThreadPool();
} else if ("Socket".equals(connectorName)) {
- final SocketConnector c = new SocketConnector();
+ final SocketConnector c = useSsl
+ ? new SslSocketConnector(sslcontext)
+ : new SocketConnector();
c.setReuseAddress(true);
connector = c;
threadPool = (QueuedThreadPool) c.getThreadPool();
} else {
throw new IllegalArgumentException("Illegal value for system property 'tests.jettyConnector': " + connectorName);
}
+
connector.setPort(port);
connector.setHost("127.0.0.1");
if (threadPool != null) {
@@ -346,6 +390,31 @@ public class JettySolrRunner {
return lastPort;
}
+ /**
+ * Returns a base URL consisting of the protocal, host, and port for a
+ * Connector in use by the Jetty Server contained in this runner.
+ */
+ public URL getBaseUrl() {
+ String protocol = null;
+ try {
+ Connector[] conns = server.getConnectors();
+ if (0 == conns.length) {
+ throw new IllegalStateException("Jetty Server has no Connectors");
+ }
+ Connector c = conns[0];
+ if (c.getLocalPort() < 0) {
+ throw new IllegalStateException("Jetty Connector is not open: " +
+ c.getLocalPort());
+ }
+ protocol = (c instanceof SslConnector) ? "https" : "http";
+ return new URL(protocol, c.getHost(), c.getLocalPort(), context);
+
+ } catch (MalformedURLException e) {
+ throw new IllegalStateException
+ ("Java could not make sense of protocol: " + protocol, e);
+ }
+ }
+
public DebugFilter getDebugFilter() {
return (DebugFilter)debugFilter.getFilter();
}
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Fri Feb 15 22:58:27 2013
@@ -185,7 +185,7 @@ public class OverseerCollectionProcessor
} catch (SolrException ex) {
SolrException.log(log, "Collection " + operation + " of " + operation
+ " failed");
- results.add("Operation " + operation + " cause exception:", ex);
+ results.add("Operation " + operation + " caused exception:", ex);
} finally {
return new OverseerSolrResponse(results);
}