You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2012/10/25 15:10:51 UTC
svn commit: r1402140 [12/17] - in /lucene/dev/branches/LUCENE-2878: ./
dev-tools/ dev-tools/eclipse/ dev-tools/eclipse/dot.settings/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/classification/
dev-tools/maven/ dev-tools/maven/lucene/classific...
Modified: lucene/dev/branches/LUCENE-2878/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQueryCapable.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQueryCapable.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQueryCapable.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQueryCapable.java Thu Oct 25 13:10:25 2012
@@ -23,6 +23,14 @@ package org.apache.lucene.sandbox.querie
*/
public interface RegexQueryCapable {
+ /**
+ * Defines which {@link RegexCapabilities} implementation is used by this instance.
+ * @see #getRegexImplementation()
+ */
void setRegexImplementation(RegexCapabilities impl);
+ /**
+ * Returns the implementation used by this instance.
+ * @see #setRegexImplementation(RegexCapabilities)
+ */
RegexCapabilities getRegexImplementation();
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/site/changes/changes2html.pl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/site/changes/changes2html.pl?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/site/changes/changes2html.pl (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/site/changes/changes2html.pl Thu Oct 25 13:10:25 2012
@@ -41,17 +41,10 @@ my $first_relid = undef;
my $second_relid = undef;
my @releases = ();
-my @lines = <>; # Get all input at once
+my @lines = <STDIN>; # Get all input at once
-my $product = '';
-for my $line (@lines) {
- ($product) = $line =~ /(Solr|Lucene)/i;
- if ($product) {
- $product = uc($product);
- last;
- }
-}
-my %release_dates = &setup_release_dates;
+my $product = $ARGV[0];
+my %release_dates = &setup_release_dates($ARGV[1]);
my $in_major_component_versions_section = 0;
@@ -804,6 +797,7 @@ sub get_release_date {
#
sub setup_release_dates {
my %release_dates;
+ my $file = shift;
if (uc($product) eq 'LUCENE') {
%release_dates
= ( '0.01' => '2000-03-30', '0.04' => '2000-04-19',
@@ -826,9 +820,7 @@ sub setup_release_dates {
'3.0.0' => '2009-11-25');
}
- print STDERR "Retrieving $project_info_url/$product ...\n";
- my $project_info_json = get_url_contents("$project_info_url/$product");
-
+ my $project_info_json = readFile($file);
my $project_info = json2perl($project_info_json);
for my $version (@{$project_info->{versions}}) {
if ($version->{releaseDate}) {
@@ -844,20 +836,13 @@ sub setup_release_dates {
return %release_dates;
}
-#
-# returns contents of the passed in url
-#
-sub get_url_contents {
- my $url = shift;
- my $tryWget = `wget --no-check-certificate -O - $url`;
- if ($? eq 0) {
- return $tryWget;
- }
- my $tryCurl = `curl $url`;
- if ($? eq 0) {
- return $tryCurl;
- }
- die "could not retrieve $url with either wget or curl!";
+sub readFile {
+ my $file = shift;
+ open(F, '<'.$file) || die "could not open $file: $!";
+ local $/ = undef;
+ my $project_info_json = <F>;
+ close(F);
+ return $project_info_json;
}
#
Modified: lucene/dev/branches/LUCENE-2878/lucene/site/xsl/index.xsl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/site/xsl/index.xsl?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/site/xsl/index.xsl (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/site/xsl/index.xsl Thu Oct 25 13:10:25 2012
@@ -63,9 +63,10 @@
<h2>Reference Documents</h2>
<ul>
<li><a href="changes/Changes.html">Changes</a>: List of changes in this release.</li>
+ <li><a href="SYSTEM_REQUIREMENTS.html">System Requirements</a>: Minimum and supported Java versions.</li>
<li><a href="MIGRATE.html">Migration Guide</a>: What changed in Lucene 4; how to migrate code from Lucene 3.x.</li>
<li><a href="JRE_VERSION_MIGRATION.html">JRE Version Migration</a>: Information about upgrading between major JRE versions.</li>
- <li><a href="core/org/apache/lucene/codecs/lucene40/package-summary.html#package_description">File Formats</a>: Guide to the supported index format used by Lucene. This can be customized by using <a href="core/org/apache/lucene/codecs/package-summary.html#package_description">an alternate codec</a>.</li>
+ <li><a href="core/org/apache/lucene/codecs/lucene41/package-summary.html#package_description">File Formats</a>: Guide to the supported index format used by Lucene. This can be customized by using <a href="core/org/apache/lucene/codecs/package-summary.html#package_description">an alternate codec</a>.</li>
<li><a href="core/org/apache/lucene/search/package-summary.html#package_description">Search and Scoring in Lucene</a>: Introduction to how Lucene scores documents.</li>
<li><a href="core/org/apache/lucene/search/similarities/TFIDFSimilarity.html">Classic Scoring Formula</a>: Formula of Lucene's classic <a href="http://en.wikipedia.org/wiki/Vector_Space_Model">Vector Space</a> implementation. (look <a href="core/org/apache/lucene/search/similarities/package-summary.html#package_description">here</a> for other models)</li>
<li><a href="queryparser/org/apache/lucene/queryparser/classic/package-summary.html#package_description">Classic QueryParser Syntax</a>: Overview of the Classic QueryParser's syntax and features.</li>
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java Thu Oct 25 13:10:25 2012
@@ -40,12 +40,12 @@ import org.apache.lucene.spatial.query.S
* <li>What types of query shapes can be used?</li>
* <li>What types of query operations are supported?
* This might vary per shape.</li>
- * <li>Are there caches? Under what circumstances are they used?
- * Roughly how big are they? Is it segmented by Lucene segments, such as is
- * done by the Lucene {@link org.apache.lucene.search.FieldCache} and
- * {@link org.apache.lucene.index.DocValues} (ideal) or is it for the entire
- * index?
+ * <li>Does it use the {@link org.apache.lucene.search.FieldCache}, {@link
+ * org.apache.lucene.index.DocValues} or some other type of cache? When?
* </ul>
+ * If a strategy only supports certain shapes at index or query time, then in
+ * general it will throw an exception if given an incompatible one. It will not
+ * be coerced into compatibility.
* <p/>
* Note that a SpatialStrategy is not involved with the Lucene stored field
* values of shapes, which is immaterial to indexing & search.
@@ -85,7 +85,7 @@ public abstract class SpatialStrategy {
}
/**
- * Returns the IndexableField(s) from the <code>shape</code> that are to be
+ * Returns the IndexableField(s) from the {@code shape} that are to be
* added to the {@link org.apache.lucene.document.Document}. These fields
* are expected to be marked as indexed and not stored.
* <p/>
@@ -96,6 +96,7 @@ public abstract class SpatialStrategy {
* since it doesn't use it.
*
* @return Not null nor will it have null elements.
+ * @throws UnsupportedOperationException if given a shape incompatible with the strategy
*/
public abstract Field[] createIndexableFields(Shape shape);
@@ -111,6 +112,10 @@ public abstract class SpatialStrategy {
* and {@link Shape} from the supplied {@code args}.
* The default implementation is
* <pre>return new ConstantScoreQuery(makeFilter(args));</pre>
+ *
+ * @throws UnsupportedOperationException If the strategy does not support the shape in {@code args}
+ * @throws org.apache.lucene.spatial.query.UnsupportedSpatialOperation If the strategy does not support the {@link
+ * org.apache.lucene.spatial.query.SpatialOperation} in {@code args}.
*/
public ConstantScoreQuery makeQuery(SpatialArgs args) {
return new ConstantScoreQuery(makeFilter(args));
@@ -124,13 +129,17 @@ public abstract class SpatialStrategy {
* {@link #makeQuery(org.apache.lucene.spatial.query.SpatialArgs)}
* then this method could be simply:
* <pre>return new QueryWrapperFilter(makeQuery(args).getQuery());</pre>
+ *
+ * @throws UnsupportedOperationException If the strategy does not support the shape in {@code args}
+ * @throws org.apache.lucene.spatial.query.UnsupportedSpatialOperation If the strategy does not support the {@link
+ * org.apache.lucene.spatial.query.SpatialOperation} in {@code args}.
*/
public abstract Filter makeFilter(SpatialArgs args);
/**
* Returns a ValueSource with values ranging from 1 to 0, depending inversely
* on the distance from {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point)}.
- * The formula is <code>c/(d + c)</code> where 'd' is the distance and 'c' is
+ * The formula is {@code c/(d + c)} where 'd' is the distance and 'c' is
* one tenth the distance to the farthest edge from the center. Thus the
* scores will be 1 for indexed points at the center of the query shape and as
* low as ~0.1 at its furthest edges.
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java Thu Oct 25 13:10:25 2012
@@ -43,8 +43,28 @@ import org.apache.lucene.spatial.query.U
/**
- * Based on GeoPortal's
- * <a href="http://geoportal.svn.sourceforge.net/svnroot/geoportal/Geoportal/trunk/src/com/esri/gpt/catalog/lucene/SpatialClauseAdapter.java">SpatialClauseAdapter</a>.
+ * A SpatialStrategy for indexing and searching Rectangles by storing its
+ * coordinates in numeric fields. It supports all {@link SpatialOperation}s and
+ * has a custom overlap relevancy. It is based on GeoPortal's <a
+ * href="http://geoportal.svn.sourceforge.net/svnroot/geoportal/Geoportal/trunk/src/com/esri/gpt/catalog/lucene/SpatialClauseAdapter.java">SpatialClauseAdapter</a>.
+ *
+ * <h4>Characteristics:</h4>
+ * <ul>
+ * <li>Only indexes Rectangles; just one per field value.</li>
+ * <li>Can query only by a Rectangle.</li>
+ * <li>Supports all {@link SpatialOperation}s.</li>
+ * <li>Uses the FieldCache for any sorting / relevancy.</li>
+ * </ul>
+ *
+ * <h4>Implementation:</h4>
+ * This uses 4 double fields for minX, maxX, minY, maxY
+ * and a boolean to mark a dateline cross. Depending on the particular {@link
+ * SpatialOperation}s, there is a variety of {@link NumericRangeQuery}s to be
+ * done.
+ * The {@link #makeBBoxAreaSimilarityValueSource(com.spatial4j.core.shape.Rectangle)}
+ * works by calculating the query bbox overlap percentage against the indexed
+ * shape overlap percentage. The indexed shape's coordinates are retrieved from
+ * the {@link org.apache.lucene.search.FieldCache}.
*
* @lucene.experimental
*/
@@ -95,7 +115,7 @@ public class BBoxStrategy extends Spatia
public Field[] createIndexableFields(Shape shape) {
if (shape instanceof Rectangle)
return createIndexableFields((Rectangle)shape);
- throw new IllegalArgumentException("Can only index Rectangle, not " + shape);
+ throw new UnsupportedOperationException("Can only index Rectangle, not " + shape);
}
public Field[] createIndexableFields(Rectangle bbox) {
@@ -150,7 +170,7 @@ public class BBoxStrategy extends Spatia
private Query makeSpatialQuery(SpatialArgs args) {
Shape shape = args.getShape();
if (!(shape instanceof Rectangle))
- throw new IllegalArgumentException("Can only query by Rectangle, not " + shape);
+ throw new UnsupportedOperationException("Can only query by Rectangle, not " + shape);
Rectangle bbox = (Rectangle) shape;
Query spatial = null;
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java Thu Oct 25 13:10:25 2012
@@ -40,8 +40,7 @@ public class PointPrefixTreeFieldCachePr
this.grid = grid;
}
- //A kluge that this is a field
- private Node scanCell = null;
+ private Node scanCell = null;//re-used in readShape to save GC
@Override
protected Point readShape(BytesRef term) {
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java Thu Oct 25 13:10:25 2012
@@ -37,8 +37,41 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
- * Abstract SpatialStrategy which provides common functionality for those
- * Strategys which use {@link SpatialPrefixTree}s
+ * An abstract SpatialStrategy based on {@link SpatialPrefixTree}. The two
+ * subclasses are {@link RecursivePrefixTreeStrategy} and {@link
+ * TermQueryPrefixTreeStrategy}. This strategy is most effective as a fast
+ * approximate spatial search filter.
+ *
+ * <h4>Characteristics:</h4>
+ * <ul>
+ * <li>Can index any shape; however only {@link RecursivePrefixTreeStrategy}
+ * can effectively search non-point shapes. <em>Not tested.</em></li>
+ * <li>Can index a variable number of shapes per field value. This strategy
+ * can do it via multiple calls to {@link #createIndexableFields(com.spatial4j.core.shape.Shape)}
+ * for a document or by giving it some sort of Shape aggregate (e.g. JTS
+ * WKT MultiPoint). The shape's boundary is approximated to a grid precision.
+ * </li>
+ * <li>Can query with any shape. The shape's boundary is approximated to a grid
+ * precision.</li>
+ * <li>Only {@link org.apache.lucene.spatial.query.SpatialOperation#Intersects}
+ * is supported. If only points are indexed then this is effectively equivalent
+ * to IsWithin.</li>
+ * <li>The strategy supports {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point)}
+ * even for multi-valued data. However, <em>it will likely be removed in the
+ * future</em> in lieu of using another strategy with a more scalable
+ * implementation. Use of this call is the only
+ * circumstance in which a cache is used. The cache is simple but as such
+ * it doesn't scale to large numbers of points nor is it real-time-search
+ * friendly.</li>
+ * </ul>
+ *
+ * <h4>Implementation:</h4>
+ * The {@link SpatialPrefixTree} does most of the work, for example returning
+ * a list of terms representing grids of various sizes for a supplied shape.
+ * An important
+ * configuration item is {@link #setDistErrPct(double)} which balances
+ * shape precision against scalability. See those javadocs.
+ *
* @lucene.internal
*/
public abstract class PrefixTreeStrategy extends SpatialStrategy {
@@ -52,7 +85,12 @@ public abstract class PrefixTreeStrategy
this.grid = grid;
}
- /** Used in the in-memory ValueSource as a default ArrayList length for this field's array of values, per doc. */
+ /**
+ * A memory hint used by {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point)}
+ * for how big the initial size of each Document's array should be. The
+ * default is 2. Set this to slightly more than the default expected number
+ * of points per document.
+ */
public void setDefaultFieldValuesArrayLen(int defaultFieldValuesArrayLen) {
this.defaultFieldValuesArrayLen = defaultFieldValuesArrayLen;
}
@@ -62,8 +100,14 @@ public abstract class PrefixTreeStrategy
}
/**
- * The default measure of shape precision affecting indexed and query shapes.
- * Specific shapes at index and query time can use something different.
+ * The default measure of shape precision affecting shapes at index and query
+ * times. Points don't use this as they are always indexed at the configured
+ * maximum precision ({@link org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree#getMaxLevels()});
+ * this applies to all other shapes. Specific shapes at index and query time
+ * can use something different than this default value. If you don't set a
+ * default then the default is {@link SpatialArgs#DEFAULT_DISTERRPCT} --
+ * 2.5%.
+ *
* @see org.apache.lucene.spatial.query.SpatialArgs#getDistErrPct()
*/
public void setDistErrPct(double distErrPct) {
@@ -81,7 +125,8 @@ public abstract class PrefixTreeStrategy
List<Node> cells = grid.getNodes(shape, detailLevel, true);//true=intermediates cells
//If shape isn't a point, add a full-resolution center-point so that
// PointPrefixTreeFieldCacheProvider has the center-points.
- // TODO index each center of a multi-point? Yes/no?
+ //TODO index each point of a multi-point or other aggregate.
+ //TODO remove this once support for a distance ValueSource is removed.
if (!(shape instanceof Point)) {
Point ctr = shape.getCenter();
//TODO should be smarter; don't index 2 tokens for this in CellTokenStream. Harmless though.
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeFilter.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeFilter.java Thu Oct 25 13:10:25 2012
@@ -34,11 +34,15 @@ import java.io.IOException;
import java.util.LinkedList;
/**
- * Performs a spatial intersection filter against a field indexed with {@link SpatialPrefixTree}, a Trie.
- * SPT yields terms (grids) at length 1 and at greater lengths corresponding to greater precisions.
- * This filter recursively traverses each grid length and uses methods on {@link Shape} to efficiently know
- * that all points at a prefix fit in the shape or not to either short-circuit unnecessary traversals or to efficiently
- * load all enclosed points.
+ * Performs a spatial intersection filter between a query shape and a field
+ * indexed with {@link SpatialPrefixTree}, a Trie. SPT yields terms (grids) at
+ * length 1 (aka "Level 1") and at greater lengths corresponding to greater
+ * precisions. This filter recursively traverses each grid length and uses
+ * methods on {@link Shape} to efficiently know that all points at a prefix fit
+ * in the shape or not to either short-circuit unnecessary traversals or to
+ * efficiently load all enclosed points. If no indexed data lies in a portion
+ * of the shape then that portion of the query shape is quickly passed over
+ * without decomposing the shape unnecessarily.
*
* @lucene.internal
*/
@@ -165,7 +169,7 @@ RE "scan" threshold:
@Override
public String toString() {
- return "GeoFilter{fieldName='" + fieldName + '\'' + ", shape=" + queryShape + '}';
+ return getClass().getSimpleName()+"{fieldName='" + fieldName + '\'' + ", shape=" + queryShape + '}';
}
@Override
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java Thu Oct 25 13:10:25 2012
@@ -25,7 +25,11 @@ import org.apache.lucene.spatial.query.S
import org.apache.lucene.spatial.query.UnsupportedSpatialOperation;
/**
- * Based on {@link RecursivePrefixTreeFilter}.
+ * A {@link PrefixTreeStrategy} which uses {@link RecursivePrefixTreeFilter}.
+ * This strategy has support for searching non-point shapes (note: not tested).
+ * Even a query shape with distErrPct=0 (fully precise to the grid) should have
+ * good performance for typical data, unless there is a lot of indexed data
+ * coincident with the shape's edge.
*
* @lucene.experimental
*/
@@ -38,6 +42,13 @@ public class RecursivePrefixTreeStrategy
prefixGridScanLevel = grid.getMaxLevels() - 4;//TODO this default constant is dependent on the prefix grid size
}
+ /**
+ * Sets the grid level [1-maxLevels] at which indexed terms are scanned brute-force
+ * instead of by grid decomposition. By default this is maxLevels - 4. The
+ * final level, maxLevels, is always scanned.
+ *
+ * @param prefixGridScanLevel 1 to maxLevels
+ */
public void setPrefixGridScanLevel(int prefixGridScanLevel) {
//TODO if negative then subtract from maxlevels
this.prefixGridScanLevel = prefixGridScanLevel;
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java Thu Oct 25 13:10:25 2012
@@ -30,8 +30,13 @@ import org.apache.lucene.spatial.query.U
import java.util.List;
/**
- * A basic implementation using a large {@link TermsFilter} of all the nodes from
- * {@link SpatialPrefixTree#getNodes(com.spatial4j.core.shape.Shape, int, boolean)}.
+ * A basic implementation of {@link PrefixTreeStrategy} using a large {@link
+ * TermsFilter} of all the nodes from {@link SpatialPrefixTree#getNodes(com.spatial4j.core.shape.Shape,
+ * int, boolean)}. It only supports the search of indexed Point shapes.
+ * <p/>
+ * The precision of query shapes (distErrPct) is an important factor in using
+ * this Strategy. If the precision is too precise then it will result in many
+ * terms which will amount to a slower query.
*
* @lucene.experimental
*/
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java Thu Oct 25 13:10:25 2012
@@ -29,7 +29,9 @@ import java.util.List;
/**
- * A SpatialPrefixGrid based on Geohashes. Uses {@link GeohashUtils} to do all the geohash work.
+ * A {@link SpatialPrefixTree} based on
+ * <a href="http://en.wikipedia.org/wiki/Geohash">Geohashes</a>.
+ * Uses {@link GeohashUtils} to do all the geohash work.
*
* @lucene.experimental
*/
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java Thu Oct 25 13:10:25 2012
@@ -27,7 +27,7 @@ import java.util.Collections;
import java.util.List;
/**
- * Represents a grid cell. These are not necessarily threadsafe, although new Cell("") (world cell) must be.
+ * Represents a grid cell. These are not necessarily thread-safe, although new Cell("") (world cell) must be.
*
* @lucene.experimental
*/
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java Thu Oct 25 13:10:25 2012
@@ -31,8 +31,9 @@ import java.util.List;
import java.util.Locale;
/**
- * Implementation of {@link SpatialPrefixTree} which uses a quad tree
- * (http://en.wikipedia.org/wiki/Quadtree)
+ * A {@link SpatialPrefixTree} which uses a
+ * <a href="http://en.wikipedia.org/wiki/Quadtree">quad tree</a> in which an
+ * indexed term will be generated for each node, 'A', 'B', 'C', 'D'.
*
* @lucene.experimental
*/
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java Thu Oct 25 13:10:25 2012
@@ -28,10 +28,13 @@ import java.util.Collections;
import java.util.List;
/**
- * A spatial Prefix Tree, or Trie, which decomposes shapes into prefixed strings at variable lengths corresponding to
- * variable precision. Each string corresponds to a spatial region.
- *
- * Implementations of this class should be thread-safe and immutable once initialized.
+ * A spatial Prefix Tree, or Trie, which decomposes shapes into prefixed strings
+ * at variable lengths corresponding to variable precision. Each string
+ * corresponds to a rectangular spatial region. This approach is
+ * also referred to "Grids", "Tiles", and "Spatial Tiers".
+ * <p/>
+ * Implementations of this class should be thread-safe and immutable once
+ * initialized.
*
* @lucene.experimental
*/
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java Thu Oct 25 13:10:25 2012
@@ -31,6 +31,9 @@ import java.util.Map;
public abstract class SpatialPrefixTreeFactory {
private static final double DEFAULT_GEO_MAX_DETAIL_KM = 0.001;//1m
+ public static final String PREFIX_TREE = "prefixTree";
+ public static final String MAX_LEVELS = "maxLevels";
+ public static final String MAX_DIST_ERR = "maxDistErr";
protected Map<String, String> args;
protected SpatialContext ctx;
@@ -42,7 +45,7 @@ public abstract class SpatialPrefixTreeF
*/
public static SpatialPrefixTree makeSPT(Map<String,String> args, ClassLoader classLoader, SpatialContext ctx) {
SpatialPrefixTreeFactory instance;
- String cname = args.get("prefixTree");
+ String cname = args.get(PREFIX_TREE);
if (cname == null)
cname = ctx.isGeo() ? "geohash" : "quad";
if ("geohash".equalsIgnoreCase(cname))
@@ -68,14 +71,14 @@ public abstract class SpatialPrefixTreeF
}
protected void initMaxLevels() {
- String mlStr = args.get("maxLevels");
+ String mlStr = args.get(MAX_LEVELS);
if (mlStr != null) {
maxLevels = Integer.valueOf(mlStr);
return;
}
double degrees;
- String maxDetailDistStr = args.get("maxDetailDist");
+ String maxDetailDistStr = args.get(MAX_DIST_ERR);
if (maxDetailDistStr == null) {
if (!ctx.isGeo()) {
return;//let default to max
@@ -83,9 +86,6 @@ public abstract class SpatialPrefixTreeF
degrees = DistanceUtils.dist2Degrees(DEFAULT_GEO_MAX_DETAIL_KM, DistanceUtils.EARTH_MEAN_RADIUS_KM);
} else {
degrees = Double.parseDouble(maxDetailDistStr);
- if (ctx.isGeo()) {
- degrees = DistanceUtils.dist2Degrees(Double.parseDouble(maxDetailDistStr), DistanceUtils.EARTH_MEAN_RADIUS_KM);
- }
}
maxLevels = getLevelForDistance(degrees);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java Thu Oct 25 13:10:25 2012
@@ -24,6 +24,7 @@ import com.spatial4j.core.shape.Shape;
/**
* Principally holds the query {@link Shape} and the {@link SpatialOperation}.
+ * It's used as an argument to some methods on {@link org.apache.lucene.spatial.SpatialStrategy}.
*
* @lucene.experimental
*/
@@ -72,7 +73,6 @@ public class SpatialArgs {
* Gets the error distance that specifies how precise the query shape is. This
* looks at {@link #getDistErr()}, {@link #getDistErrPct()}, and {@code
* defaultDistErrPct}.
- * @param ctx
* @param defaultDistErrPct 0 to 0.5
* @return >= 0
*/
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java Thu Oct 25 13:10:25 2012
@@ -34,7 +34,7 @@ import java.util.StringTokenizer;
* ShapeReadWriter#readShape(String)}. There are some optional name-value pair parameters
* that follow the closing parenthesis. Example:
* <pre>
- * Intersects(-10,20,-8,22) distPec=0.025
+ * Intersects(-10,20,-8,22) distErrPct=0.025
* </pre>
* <p/>
* In the future it would be good to support something at least semi-standardized like a
@@ -86,7 +86,7 @@ public class SpatialArgsParser {
throw new IllegalArgumentException("missing body : " + v, null);
}
- Shape shape = new ShapeReadWriter(ctx).readShape(body);
+ Shape shape = ctx.readShape(body);
SpatialArgs args = new SpatialArgs(op, shape);
if (v.length() > (edx + 1)) {
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/UnsupportedSpatialOperation.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/UnsupportedSpatialOperation.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/UnsupportedSpatialOperation.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/UnsupportedSpatialOperation.java Thu Oct 25 13:10:25 2012
@@ -18,7 +18,7 @@ package org.apache.lucene.spatial.query;
*/
/**
- * Exception thrown when the strategy cannot implement the requested operation.
+ * Exception thrown when the {@link org.apache.lucene.spatial.SpatialStrategy} cannot implement the requested operation.
* @lucene.experimental
*/
public class UnsupportedSpatialOperation extends UnsupportedOperationException {
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java Thu Oct 25 13:10:25 2012
@@ -24,7 +24,10 @@ import java.util.List;
/**
* Bounded Cache of Shapes associated with docIds. Note, multiple Shapes can be
- * associated with a given docId
+ * associated with a given docId.
+ * <p>
+ * WARNING: This class holds the data in an extremely inefficient manner as all Points are in memory as objects and they
+ * are stored in many ArrayLists (one per document). So it works but doesn't scale. It will be replaced in the future.
*
* @lucene.internal
*/
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ValueSourceFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ValueSourceFilter.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ValueSourceFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ValueSourceFilter.java Thu Oct 25 13:10:25 2012
@@ -28,11 +28,12 @@ import org.apache.lucene.util.Bits;
import java.io.IOException;
/**
- * Filter that matches all documents where a valuesource is
+ * Filter that matches all documents where a ValueSource is
* in between a range of <code>min</code> and <code>max</code> inclusive.
* @lucene.internal
*/
public class ValueSourceFilter extends Filter {
+ //TODO see https://issues.apache.org/jira/browse/LUCENE-4251 (move out of spatial & improve)
final Filter startingFilter;
final ValueSource source;
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java Thu Oct 25 13:10:25 2012
@@ -30,19 +30,20 @@ import java.io.IOException;
import java.util.Map;
/**
- * An implementation of the Lucene ValueSource model that returns the distance.
+ * An implementation of the Lucene ValueSource model that returns the distance
+ * for a {@link PointVectorStrategy}.
*
* @lucene.internal
*/
public class DistanceValueSource extends ValueSource {
- private TwoDoublesStrategy strategy;
+ private PointVectorStrategy strategy;
private final Point from;
/**
* Constructor.
*/
- public DistanceValueSource(TwoDoublesStrategy strategy, Point from) {
+ public DistanceValueSource(PointVectorStrategy strategy, Point from) {
this.strategy = strategy;
this.from = from;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java Thu Oct 25 13:10:25 2012
@@ -29,7 +29,7 @@ import org.apache.lucene.spatial.prefix.
import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
-import org.apache.lucene.spatial.vector.TwoDoublesStrategy;
+import org.apache.lucene.spatial.vector.PointVectorStrategy;
import org.junit.Test;
import java.io.IOException;
@@ -54,7 +54,7 @@ public class DistanceStrategyTest extend
strategy = new TermQueryPrefixTreeStrategy(grid, "termquery_geohash");
ctorArgs.add(new Object[]{new Param(strategy)});
- strategy = new TwoDoublesStrategy(ctx, "twodoubles");
+ strategy = new PointVectorStrategy(ctx, "pointvector");
ctorArgs.add(new Object[]{new Param(strategy)});
strategy = new BBoxStrategy(ctx, "bbox");
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java Thu Oct 25 13:10:25 2012
@@ -21,7 +21,6 @@ import com.carrotsearch.randomizedtestin
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.spatial4j.core.context.SpatialContext;
import com.spatial4j.core.distance.DistanceUtils;
-import com.spatial4j.core.io.ShapeReadWriter;
import com.spatial4j.core.shape.Point;
import com.spatial4j.core.shape.Shape;
import org.apache.lucene.search.FilteredQuery;
@@ -34,7 +33,7 @@ import org.apache.lucene.spatial.prefix.
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.query.SpatialOperation;
-import org.apache.lucene.spatial.vector.TwoDoublesStrategy;
+import org.apache.lucene.spatial.vector.PointVectorStrategy;
import org.junit.Test;
import java.io.IOException;
@@ -68,7 +67,7 @@ public class PortedSolr3Test extends Str
strategy = new TermQueryPrefixTreeStrategy(grid, "termquery_geohash");
ctorArgs.add(new Object[]{new Param(strategy)});
- strategy = new TwoDoublesStrategy(ctx, "twodoubles");
+ strategy = new PointVectorStrategy(ctx, "pointvector");
ctorArgs.add(new Object[]{new Param(strategy)});
return ctorArgs;
@@ -162,7 +161,7 @@ public class PortedSolr3Test extends Str
private void _checkHits(boolean bbox, String ptStr, double distKM, int assertNumFound, int... assertIds) {
SpatialOperation op = SpatialOperation.Intersects;
- Point pt = (Point) new ShapeReadWriter(ctx).readShape(ptStr);
+ Point pt = (Point) ctx.readShape(ptStr);
double distDEG = DistanceUtils.dist2Degrees(distKM, DistanceUtils.EARTH_MEAN_RADIUS_KM);
Shape shape = ctx.makeCircle(pt, distDEG);
if (bbox)
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java Thu Oct 25 13:10:25 2012
@@ -19,7 +19,6 @@ package org.apache.lucene.spatial;
import com.spatial4j.core.context.SpatialContext;
import com.spatial4j.core.distance.DistanceUtils;
-import com.spatial4j.core.io.ShapeReadWriter;
import com.spatial4j.core.shape.Point;
import com.spatial4j.core.shape.Shape;
import org.apache.lucene.document.Document;
@@ -111,7 +110,7 @@ public class SpatialExample extends Luce
//When parsing a string to a shape, the presence of a comma means it's y-x
// order (lon, lat)
indexWriter.addDocument(newSampleDocument(
- 4, new ShapeReadWriter(ctx).readShape("-50.7693246, 60.9289094")));
+ 4, ctx.readShape("-50.7693246, 60.9289094")));
indexWriter.addDocument(newSampleDocument(
20, ctx.makePoint(0.1,0.1), ctx.makePoint(0, 0)));
Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java Thu Oct 25 13:10:25 2012
@@ -19,7 +19,6 @@ package org.apache.lucene.spatial;
*/
import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.io.ShapeReadWriter;
import com.spatial4j.core.io.sample.SampleData;
import com.spatial4j.core.io.sample.SampleDataReader;
import com.spatial4j.core.shape.Shape;
@@ -35,6 +34,7 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.spatial.query.SpatialArgsParser;
import org.junit.Assert;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
@@ -89,7 +89,7 @@ public abstract class StrategyTestCase e
Document document = new Document();
document.add(new StringField("id", data.id, Field.Store.YES));
document.add(new StringField("name", data.name, Field.Store.YES));
- Shape shape = new ShapeReadWriter(ctx).readShape(data.shape);
+ Shape shape = ctx.readShape(data.shape);
shape = convertShapeFromGetDocuments(shape);
if (shape != null) {
for (Field f : strategy.createIndexableFields(shape)) {
@@ -110,8 +110,11 @@ public abstract class StrategyTestCase e
}
protected Iterator<SampleData> getSampleData(String testDataFile) throws IOException {
- return new SampleDataReader(
- getClass().getClassLoader().getResourceAsStream("data/"+testDataFile) );
+ String path = "data/" + testDataFile;
+ InputStream stream = getClass().getClassLoader().getResourceAsStream(path);
+ if (stream == null)
+ throw new FileNotFoundException("classpath resource not found: "+path);
+ return new SampleDataReader(stream);
}
protected Iterator<SpatialTestQuery> getTestQueries(String testQueryFile, SpatialContext ctx) throws IOException {
@@ -130,7 +133,7 @@ public abstract class StrategyTestCase e
SearchResults got = executeQuery(strategy.makeQuery(q.args), 100);
if (storeShape && got.numFound > 0) {
//check stored value is there & parses
- assertNotNull(new ShapeReadWriter(ctx).readShape(got.results.get(0).document.get(strategy.getFieldName())));
+ assertNotNull(ctx.readShape(got.results.get(0).document.get(strategy.getFieldName())));
}
if (concern.orderIsImportant) {
Iterator<String> ids = q.ids.iterator();
@@ -174,7 +177,7 @@ public abstract class StrategyTestCase e
}
protected void adoc(String id, String shapeStr) throws IOException {
- Shape shape = shapeStr==null ? null : new ShapeReadWriter(ctx).readShape(shapeStr);
+ Shape shape = shapeStr==null ? null : ctx.readShape(shapeStr);
addDocument(newDoc(id, shape));
}
protected void adoc(String id, Shape shape) throws IOException {
Modified: lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java Thu Oct 25 13:10:25 2012
@@ -313,7 +313,7 @@ public class DirectSpellChecker {
* @param suggestMode specifies when to return suggested words
* @param accuracy return only suggested words that match with this similarity
* @return sorted list of the suggested words according to the comparator
- * @throws IOException
+ * @throws IOException If there is a low-level I/O error.
*/
public SuggestWord[] suggestSimilar(Term term, int numSug, IndexReader ir,
SuggestMode suggestMode, float accuracy) throws IOException {
Modified: lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java Thu Oct 25 13:10:25 2012
@@ -449,7 +449,7 @@ public class SpellChecker implements jav
/**
* Removes all terms from the spell check index.
- * @throws IOException
+ * @throws IOException If there is a low-level I/O error.
* @throws AlreadyClosedException if the Spellchecker is already closed
*/
public void clearIndex() throws IOException {
@@ -467,8 +467,8 @@ public class SpellChecker implements jav
/**
* Check whether the word exists in the index.
- * @param word
- * @throws IOException
+ * @param word word to check
+ * @throws IOException If there is a low-level I/O error.
* @throws AlreadyClosedException if the Spellchecker is already closed
* @return true if the word exists in the index
*/
@@ -490,7 +490,7 @@ public class SpellChecker implements jav
* @param config {@link IndexWriterConfig} to use
* @param fullMerge whether or not the spellcheck index should be fully merged
* @throws AlreadyClosedException if the Spellchecker is already closed
- * @throws IOException
+ * @throws IOException If there is a low-level I/O error.
*/
public final void indexDictionary(Dictionary dict, IndexWriterConfig config, boolean fullMerge) throws IOException {
synchronized (modifyCurrentIndexLock) {
Modified: lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java Thu Oct 25 13:10:25 2012
@@ -82,16 +82,13 @@ public class WordBreakSpellChecker {
* lower score is generally preferred over a higher score.
* </p>
*
- * @param term
- * @param maxSuggestions
- * @param ir
* @param suggestMode
* - default = {@link SuggestMode#SUGGEST_WHEN_NOT_IN_INDEX}
* @param sortMethod
* - default =
* {@link BreakSuggestionSortMethod#NUM_CHANGES_THEN_MAX_FREQUENCY}
* @return one or more arrays of words formed by breaking up the original term
- * @throws IOException
+ * @throws IOException If there is a low-level I/O error.
*/
public SuggestWord[][] suggestWordBreaks(Term term, int maxSuggestions,
IndexReader ir, SuggestMode suggestMode,
@@ -159,12 +156,8 @@ public class WordBreakSpellChecker {
* included term.
* </p>
*
- * @param terms
- * @param maxSuggestions
- * @param ir
- * @param suggestMode
* @return an array of words generated by combining original terms
- * @throws IOException
+ * @throws IOException If there is a low-level I/O error.
*/
public CombineSuggestion[] suggestWordCombinations(Term[] terms,
int maxSuggestions, IndexReader ir, SuggestMode suggestMode)
Modified: lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java Thu Oct 25 13:10:25 2012
@@ -56,7 +56,6 @@ import org.apache.lucene.util.fst.Util.M
* Input weights must be between 0 and {@link Integer#MAX_VALUE}, any
* other values will be rejected.
*
- * @see Util#shortestPaths(FST, FST.Arc, Comparator, int)
* @lucene.experimental
*/
public class WFSTCompletionLookup extends Lookup {
@@ -144,6 +143,11 @@ public class WFSTCompletionLookup extend
@Override
public List<LookupResult> lookup(CharSequence key, boolean onlyMorePopular, int num) {
assert num > 0;
+
+ if (onlyMorePopular) {
+ throw new IllegalArgumentException("this suggester only works with onlyMorePopular=false");
+ }
+
BytesRef scratch = new BytesRef(key);
int prefixLength = scratch.length;
Arc<Long> arc = new Arc<Long>();
@@ -168,12 +172,14 @@ public class WFSTCompletionLookup extend
return results; // that was quick
}
}
-
+
// complete top-N
MinResult<Long> completions[] = null;
try {
- completions = Util.shortestPaths(fst, arc, weightComparator, num);
- } catch (IOException bogus) { throw new RuntimeException(bogus); }
+ completions = Util.shortestPaths(fst, arc, prefixOutput, weightComparator, num, !exactFirst);
+ } catch (IOException bogus) {
+ throw new RuntimeException(bogus);
+ }
BytesRef suffix = new BytesRef(8);
for (MinResult<Long> completion : completions) {
@@ -183,7 +189,7 @@ public class WFSTCompletionLookup extend
scratch.append(suffix);
spare.grow(scratch.length);
UnicodeUtil.UTF8toUTF16(scratch, spare);
- results.add(new LookupResult(spare.toString(), decodeWeight(prefixOutput + completion.output)));
+ results.add(new LookupResult(spare.toString(), decodeWeight(completion.output)));
}
return results;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java Thu Oct 25 13:10:25 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.search.suggest
import java.io.BufferedReader;
import java.io.InputStreamReader;
+import java.lang.reflect.Constructor;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
@@ -30,7 +31,11 @@ import java.util.Random;
import java.util.concurrent.Callable;
import org.apache.lucene.util.*;
-import org.apache.lucene.search.suggest.Lookup;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.search.suggest.Lookup; // javadocs
+import org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester;
import org.apache.lucene.search.suggest.fst.FSTCompletionLookup;
import org.apache.lucene.search.suggest.fst.WFSTCompletionLookup;
import org.apache.lucene.search.suggest.jaspell.JaspellLookup;
@@ -49,7 +54,8 @@ public class LookupBenchmarkTest extends
JaspellLookup.class,
TSTLookup.class,
FSTCompletionLookup.class,
- WFSTCompletionLookup.class);
+ WFSTCompletionLookup.class,
+ AnalyzingSuggester.class);
private final static int rounds = 15;
private final static int warmup = 5;
@@ -133,10 +139,19 @@ public class LookupBenchmarkTest extends
System.err.println("-- RAM consumption");
for (Class<? extends Lookup> cls : benchmarkClasses) {
Lookup lookup = buildLookup(cls, dictionaryInput);
+ long sizeInBytes;
+ if (lookup instanceof AnalyzingSuggester) {
+ // Just get size of FST: else we are also measuring
+ // size of MockAnalyzer which is non-trivial and
+ // varies depending on test seed:
+ sizeInBytes = ((AnalyzingSuggester) lookup).sizeInBytes();
+ } else {
+ sizeInBytes = RamUsageEstimator.sizeOf(lookup);
+ }
System.err.println(
String.format(Locale.ROOT, "%-15s size[B]:%,13d",
lookup.getClass().getSimpleName(),
- RamUsageEstimator.sizeOf(lookup)));
+ sizeInBytes));
}
}
@@ -144,7 +159,13 @@ public class LookupBenchmarkTest extends
* Create {@link Lookup} instance and populate it.
*/
private Lookup buildLookup(Class<? extends Lookup> cls, TermFreq[] input) throws Exception {
- Lookup lookup = cls.newInstance();
+ Lookup lookup = null;
+ try {
+ lookup = cls.newInstance();
+ } catch (InstantiationException e) {
+ Constructor<? extends Lookup> ctor = cls.getConstructor(Analyzer.class);
+ lookup = ctor.newInstance(new MockAnalyzer(random, MockTokenizer.KEYWORD, false));
+ }
lookup.build(new TermFreqArrayIterator(input));
return lookup;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java Thu Oct 25 13:10:25 2012
@@ -44,7 +44,13 @@ public class WFSTCompletionTest extends
assertEquals(1, results.size());
assertEquals("foo", results.get(0).key.toString());
assertEquals(50, results.get(0).value, 0.01F);
-
+
+ // make sure we don't get a dup exact suggestion:
+ results = suggester.lookup(_TestUtil.stringToCharSequence("foo", random), false, 2);
+ assertEquals(1, results.size());
+ assertEquals("foo", results.get(0).key.toString());
+ assertEquals(50, results.get(0).value, 0.01F);
+
// top N of 1 for 'bar': we return this even though barbar is higher
results = suggester.lookup(_TestUtil.stringToCharSequence("bar", random), false, 1);
assertEquals(1, results.size());
@@ -69,6 +75,54 @@ public class WFSTCompletionTest extends
assertEquals("barbara", results.get(2).key.toString());
assertEquals(6, results.get(2).value, 0.01F);
}
+
+ public void testExactFirst() throws Exception {
+
+ WFSTCompletionLookup suggester = new WFSTCompletionLookup(true);
+
+ suggester.build(new TermFreqArrayIterator(new TermFreq[] {
+ new TermFreq("x y", 20),
+ new TermFreq("x", 2),
+ }));
+
+ for(int topN=1;topN<4;topN++) {
+ List<LookupResult> results = suggester.lookup("x", false, topN);
+
+ assertEquals(Math.min(topN, 2), results.size());
+
+ assertEquals("x", results.get(0).key);
+ assertEquals(2, results.get(0).value);
+
+ if (topN > 1) {
+ assertEquals("x y", results.get(1).key);
+ assertEquals(20, results.get(1).value);
+ }
+ }
+ }
+
+ public void testNonExactFirst() throws Exception {
+
+ WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
+
+ suggester.build(new TermFreqArrayIterator(new TermFreq[] {
+ new TermFreq("x y", 20),
+ new TermFreq("x", 2),
+ }));
+
+ for(int topN=1;topN<4;topN++) {
+ List<LookupResult> results = suggester.lookup("x", false, topN);
+
+ assertEquals(Math.min(topN, 2), results.size());
+
+ assertEquals("x y", results.get(0).key);
+ assertEquals(20, results.get(0).value);
+
+ if (topN > 1) {
+ assertEquals("x", results.get(1).key);
+ assertEquals(2, results.get(1).value);
+ }
+ }
+ }
public void testRandom() throws Exception {
int numWords = atLeast(1000);
Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/build.xml?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/build.xml (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/build.xml Thu Oct 25 13:10:25 2012
@@ -32,19 +32,13 @@
</path>
<!--
- Specialize compile-core to depend on lucene-core compilation,
- and *not* to depend on clover; clover already includes the
- test-framework sources in each module's test instrumentation.
+ Specialize compile-core to depend on lucene-core and lucene-codecs compilation.
-->
- <target name="compile-core" depends="init,compile-lucene-core,compile-codecs"
- description="Compiles test-framework classes">
- <compile srcdir="${src.dir}" destdir="${build.dir}/classes/java">
- <classpath refid="classpath"/>
- </compile>
- <copy todir="${build.dir}/classes/java">
- <fileset dir="${src.dir}/../resources" erroronmissingdir="no"/>
- </copy>
- </target>
+ <target name="compile-core" depends="init,compile-lucene-core,compile-codecs,common.compile-core"
+ description="Compiles test-framework classes"/>
+
+ <!-- redefine the clover setup, because we dont want to run clover for the test-framework -->
+ <target name="-clover.setup" if="run.clover"/>
<target name="javadocs-core" depends="javadocs"/>
<target name="javadocs" depends="init,javadocs-lucene-core,javadocs-lucene-codecs">
Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/ivy.xml?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/ivy.xml (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/ivy.xml Thu Oct 25 13:10:25 2012
@@ -32,8 +32,8 @@
<dependency org="org.apache.ant" name="ant" rev="1.8.2" transitive="false" />
<dependency org="junit" name="junit" rev="4.10" transitive="false" conf="default->*;junit4-stdalone->*" />
- <dependency org="com.carrotsearch.randomizedtesting" name="junit4-ant" rev="2.0.1" transitive="false" conf="default->*;junit4-stdalone->*" />
- <dependency org="com.carrotsearch.randomizedtesting" name="randomizedtesting-runner" rev="2.0.1" transitive="false" conf="default->*;junit4-stdalone->*" />
+ <dependency org="com.carrotsearch.randomizedtesting" name="junit4-ant" rev="2.0.3" transitive="false" conf="default->*;junit4-stdalone->*" />
+ <dependency org="com.carrotsearch.randomizedtesting" name="randomizedtesting-runner" rev="2.0.3" transitive="false" conf="default->*;junit4-stdalone->*" />
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java Thu Oct 25 13:10:25 2012
@@ -17,14 +17,13 @@ package org.apache.lucene.codecs.asserti
* limitations under the License.
*/
-import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40Codec;
+import org.apache.lucene.codecs.lucene41.Lucene41Codec;
/**
- * Acts like {@link Lucene40Codec} but with additional asserts.
+ * Acts like {@link Lucene41Codec} but with additional asserts.
*/
public final class AssertingCodec extends FilterCodec {
@@ -32,12 +31,7 @@ public final class AssertingCodec extend
private final TermVectorsFormat vectors = new AssertingTermVectorsFormat();
public AssertingCodec() {
- super("Asserting");
- }
-
- @Override
- protected Codec delegate() {
- return Codec.forName("Lucene40");
+ super("Asserting", new Lucene41Codec());
}
@Override
Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java Thu Oct 25 13:10:25 2012
@@ -27,7 +27,7 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.TermStats;
import org.apache.lucene.codecs.TermsConsumer;
-import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat;
+import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
import org.apache.lucene.index.AssertingAtomicReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfo.IndexOptions;
@@ -38,10 +38,10 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.OpenBitSet;
/**
- * Just like {@link Lucene40PostingsFormat} but with additional asserts.
+ * Just like {@link Lucene41PostingsFormat} but with additional asserts.
*/
public final class AssertingPostingsFormat extends PostingsFormat {
- private final PostingsFormat in = new Lucene40PostingsFormat();
+ private final PostingsFormat in = new Lucene41PostingsFormat();
public AssertingPostingsFormat() {
super("Asserting");
Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java Thu Oct 25 13:10:25 2012
@@ -38,8 +38,8 @@ import org.apache.lucene.codecs.blockter
import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;
-import org.apache.lucene.codecs.lucene40.Lucene40PostingsReader;
-import org.apache.lucene.codecs.lucene40.Lucene40PostingsWriter;
+import org.apache.lucene.codecs.lucene41.Lucene41PostingsReader;
+import org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter;
import org.apache.lucene.codecs.mockintblock.MockFixedIntBlockPostingsFormat;
import org.apache.lucene.codecs.mockintblock.MockVariableIntBlockPostingsFormat;
import org.apache.lucene.codecs.mocksep.MockSingleIntFactory;
@@ -133,9 +133,17 @@ public final class MockRandomPostingsFor
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+ int minSkipInterval;
+ if (state.segmentInfo.getDocCount() > 1000000) {
+ // Test2BPostings can OOME otherwise:
+ minSkipInterval = 3;
+ } else {
+ minSkipInterval = 2;
+ }
+
// we pull this before the seed intentionally: because its not consumed at runtime
// (the skipInterval is written into postings header)
- int skipInterval = _TestUtil.nextInt(seedRandom, 2, 10);
+ int skipInterval = _TestUtil.nextInt(seedRandom, minSkipInterval, 10);
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: skipInterval=" + skipInterval);
@@ -166,7 +174,8 @@ public final class MockRandomPostingsFor
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: writing Standard postings");
}
- postingsWriter = new Lucene40PostingsWriter(state, skipInterval);
+ // TODO: randomize variables like acceptibleOverHead?!
+ postingsWriter = new Lucene41PostingsWriter(state, skipInterval);
}
if (random.nextBoolean()) {
@@ -305,7 +314,7 @@ public final class MockRandomPostingsFor
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading Standard postings");
}
- postingsReader = new Lucene40PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
+ postingsReader = new Lucene41PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
}
if (random.nextBoolean()) {
Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java Thu Oct 25 13:10:25 2012
@@ -26,8 +26,8 @@ import org.apache.lucene.codecs.FieldsPr
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene40.Lucene40PostingsReader;
-import org.apache.lucene.codecs.lucene40.Lucene40PostingsWriter;
+import org.apache.lucene.codecs.lucene41.Lucene41PostingsReader;
+import org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter;
import org.apache.lucene.codecs.pulsing.PulsingPostingsReader;
import org.apache.lucene.codecs.pulsing.PulsingPostingsWriter;
import org.apache.lucene.index.SegmentReadState;
@@ -35,7 +35,7 @@ import org.apache.lucene.index.SegmentWr
import org.apache.lucene.util.IOUtils;
/**
- * Pulsing(1, Pulsing(2, Lucene40))
+ * Pulsing(1, Pulsing(2, Lucene41))
* @lucene.experimental
*/
// TODO: if we create PulsingPostingsBaseFormat then we
@@ -55,7 +55,7 @@ public final class NestedPulsingPostings
// Terms dict
boolean success = false;
try {
- docsWriter = new Lucene40PostingsWriter(state);
+ docsWriter = new Lucene41PostingsWriter(state);
pulsingWriterInner = new PulsingPostingsWriter(2, docsWriter);
pulsingWriter = new PulsingPostingsWriter(1, pulsingWriterInner);
@@ -77,7 +77,7 @@ public final class NestedPulsingPostings
PostingsReaderBase pulsingReader = null;
boolean success = false;
try {
- docsReader = new Lucene40PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
+ docsReader = new Lucene41PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
pulsingReaderInner = new PulsingPostingsReader(docsReader);
pulsingReader = new PulsingPostingsReader(pulsingReaderInner);
FieldsProducer ret = new BlockTreeTermsReader(
Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AlcoholicMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AlcoholicMergePolicy.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AlcoholicMergePolicy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AlcoholicMergePolicy.java Thu Oct 25 13:10:25 2012
@@ -47,6 +47,7 @@ public class AlcoholicMergePolicy extend
public AlcoholicMergePolicy(TimeZone tz, Random random) {
this.calendar = new GregorianCalendar(tz, Locale.ROOT);
+ calendar.setTimeInMillis(_TestUtil.nextLong(random, 0, Long.MAX_VALUE));
this.random = random;
maxMergeSize = _TestUtil.nextInt(random, 1024*1024, Integer.MAX_VALUE);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java Thu Oct 25 13:10:25 2012
@@ -261,6 +261,9 @@ public class AssertingAtomicReader exten
assert state != DocsEnumState.START : "freq() called before nextDoc()/advance()";
assert state != DocsEnumState.FINISHED : "freq() called after NO_MORE_DOCS";
int freq = super.freq();
+ if (freq == 0) {
+ System.out.println();
+ }
assert freq > 0;
return freq;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java Thu Oct 25 13:10:25 2012
@@ -257,9 +257,6 @@ class DocHelper {
* Writes the document to the directory using a segment
* named "test"; returns the SegmentInfo describing the new
* segment
- * @param dir
- * @param doc
- * @throws IOException
*/
public static SegmentInfoPerCommit writeDoc(Random random, Directory dir, Document doc) throws IOException
{
@@ -270,11 +267,6 @@ class DocHelper {
* Writes the document to the directory using the analyzer
* and the similarity score; returns the SegmentInfo
* describing the new segment
- * @param dir
- * @param analyzer
- * @param similarity
- * @param doc
- * @throws IOException
*/
public static SegmentInfoPerCommit writeDoc(Random random, Directory dir, Analyzer analyzer, Similarity similarity, Document doc) throws IOException {
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( /* LuceneTestCase.newIndexWriterConfig(random, */
Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java Thu Oct 25 13:10:25 2012
@@ -72,8 +72,8 @@ public final class FieldFilterAtomicRead
public void document(final int docID, final StoredFieldVisitor visitor) throws IOException {
super.document(docID, new StoredFieldVisitor() {
@Override
- public void binaryField(FieldInfo fieldInfo, byte[] value, int offset, int length) throws IOException {
- visitor.binaryField(fieldInfo, value, offset, length);
+ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
+ visitor.binaryField(fieldInfo, value);
}
@Override
Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java Thu Oct 25 13:10:25 2012
@@ -19,11 +19,13 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
-import java.util.Random;
import java.util.Map;
+import java.util.Random;
+import org.apache.lucene.index.MergePolicy.MergeTrigger;
import org.apache.lucene.util._TestUtil;
/**
@@ -39,18 +41,30 @@ public class MockRandomMergePolicy exten
}
@Override
- public MergeSpecification findMerges(SegmentInfos segmentInfos) {
+ public MergeSpecification findMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos) {
MergeSpecification mergeSpec = null;
//System.out.println("MRMP: findMerges sis=" + segmentInfos);
- if (segmentInfos.size() > 1 && random.nextInt(5) == 3) {
-
- List<SegmentInfoPerCommit> segments = new ArrayList<SegmentInfoPerCommit>(segmentInfos.asList());
+ int numSegments = segmentInfos.size();
+
+ List<SegmentInfoPerCommit> segments = new ArrayList<SegmentInfoPerCommit>();
+ final Collection<SegmentInfoPerCommit> merging = writer.get().getMergingSegments();
+
+ for(SegmentInfoPerCommit sipc : segmentInfos) {
+ if (!merging.contains(sipc)) {
+ segments.add(sipc);
+ }
+ }
+
+ numSegments = segments.size();
+
+ if (numSegments > 1 && (numSegments > 30 || random.nextInt(5) == 3)) {
+
Collections.shuffle(segments, random);
// TODO: sometimes make more than 1 merge?
mergeSpec = new MergeSpecification();
- final int segsToMerge = _TestUtil.nextInt(random, 1, segmentInfos.size());
+ final int segsToMerge = _TestUtil.nextInt(random, 1, numSegments);
mergeSpec.add(new OneMerge(segments.subList(0, segsToMerge)));
}
@@ -97,7 +111,7 @@ public class MockRandomMergePolicy exten
@Override
public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos) throws IOException {
- return findMerges(segmentInfos);
+ return findMerges(null, segmentInfos);
}
@Override
Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java Thu Oct 25 13:10:25 2012
@@ -29,11 +29,10 @@ import java.util.Set;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.asserting.AssertingPostingsFormat;
-import org.apache.lucene.codecs.block.BlockPostingsFormat;
-import org.apache.lucene.codecs.bloom.TestBloomFilteredLucene40Postings;
-import org.apache.lucene.codecs.lucene40.Lucene40Codec;
-import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat;
-import org.apache.lucene.codecs.lucene40ords.Lucene40WithOrds;
+import org.apache.lucene.codecs.lucene41.Lucene41Codec;
+import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
+import org.apache.lucene.codecs.lucene41ords.Lucene41WithOrds;
+import org.apache.lucene.codecs.bloom.TestBloomFilteredLucene41Postings;
import org.apache.lucene.codecs.memory.DirectPostingsFormat;
import org.apache.lucene.codecs.memory.MemoryPostingsFormat;
import org.apache.lucene.codecs.mockintblock.MockFixedIntBlockPostingsFormat;
@@ -41,7 +40,7 @@ import org.apache.lucene.codecs.mockintb
import org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat;
import org.apache.lucene.codecs.mocksep.MockSepPostingsFormat;
import org.apache.lucene.codecs.nestedpulsing.NestedPulsingPostingsFormat;
-import org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat;
+import org.apache.lucene.codecs.pulsing.Pulsing41PostingsFormat;
import org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -55,7 +54,7 @@ import org.apache.lucene.util._TestUtil;
* documents in different orders and the test will still be deterministic
* and reproducable.
*/
-public class RandomCodec extends Lucene40Codec {
+public class RandomCodec extends Lucene41Codec {
/** Shuffled list of postings formats to use for new mappings */
private List<PostingsFormat> formats = new ArrayList<PostingsFormat>();
@@ -94,23 +93,22 @@ public class RandomCodec extends Lucene4
int lowFreqCutoff = _TestUtil.nextInt(random, 2, 100);
add(avoidCodecs,
- new Lucene40PostingsFormat(minItemsPerBlock, maxItemsPerBlock),
- new BlockPostingsFormat(minItemsPerBlock, maxItemsPerBlock),
+ new Lucene41PostingsFormat(minItemsPerBlock, maxItemsPerBlock),
new DirectPostingsFormat(LuceneTestCase.rarely(random) ? 1 : (LuceneTestCase.rarely(random) ? Integer.MAX_VALUE : maxItemsPerBlock),
LuceneTestCase.rarely(random) ? 1 : (LuceneTestCase.rarely(random) ? Integer.MAX_VALUE : lowFreqCutoff)),
- new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock),
+ new Pulsing41PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock),
// add pulsing again with (usually) different parameters
- new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock),
- //TODO as a PostingsFormat which wraps others, we should allow TestBloomFilteredLucene40Postings to be constructed
+ new Pulsing41PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock),
+ //TODO as a PostingsFormat which wraps others, we should allow TestBloomFilteredLucene41Postings to be constructed
//with a choice of concrete PostingsFormats. Maybe useful to have a generic means of marking and dealing
//with such "wrapper" classes?
- new TestBloomFilteredLucene40Postings(),
+ new TestBloomFilteredLucene41Postings(),
new MockSepPostingsFormat(),
new MockFixedIntBlockPostingsFormat(_TestUtil.nextInt(random, 1, 2000)),
new MockVariableIntBlockPostingsFormat( _TestUtil.nextInt(random, 1, 127)),
new MockRandomPostingsFormat(random),
new NestedPulsingPostingsFormat(),
- new Lucene40WithOrds(),
+ new Lucene41WithOrds(),
new SimpleTextPostingsFormat(),
new AssertingPostingsFormat(),
new MemoryPostingsFormat(true, random.nextFloat()),
Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Thu Oct 25 13:10:25 2012
@@ -126,8 +126,23 @@ public class RandomIndexWriter implement
// any forced merges:
doRandomForceMerge = r.nextBoolean();
}
+
+ private boolean addDocValuesFields = true;
+
+ /**
+ * set to false if you don't want RandomIndexWriter
+ * adding docvalues fields.
+ */
+ public void setAddDocValuesFields(boolean v) {
+ addDocValuesFields = v;
+ switchDoDocValues();
+ }
private void switchDoDocValues() {
+ if (addDocValuesFields == false) {
+ doDocValues = false;
+ return;
+ }
// randomly enable / disable docValues
doDocValues = LuceneTestCase.rarely(r);
if (LuceneTestCase.VERBOSE) {
@@ -387,10 +402,16 @@ public class RandomIndexWriter implement
final int segCount = w.getSegmentCount();
if (r.nextBoolean() || segCount == 0) {
// full forceMerge
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("RIW: doRandomForceMerge(1)");
+ }
w.forceMerge(1);
} else {
// partial forceMerge
final int limit = _TestUtil.nextInt(r, 1, segCount);
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("RIW: doRandomForceMerge(" + limit + ")");
+ }
w.forceMerge(limit);
assert !doRandomForceMergeAssert || w.getSegmentCount() <= limit: "limit=" + limit + " actual=" + w.getSegmentCount();
}