You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/02/11 17:42:47 UTC
[13/31] lucene-solr git commit: Merge branch 'master' into
nrt_replicas
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd6804bc/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java
----------------------------------------------------------------------
diff --cc lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java
index 0000000,aa1da81..f2f8de7
mode 000000,100644..100644
--- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java
+++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java
@@@ -1,0 -1,760 +1,760 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.lucene.spatial.util;
+
+ import java.io.IOException;
+ import java.text.DecimalFormat;
+ import java.text.DecimalFormatSymbols;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.Locale;
+ import java.util.Set;
+ import java.util.concurrent.CountDownLatch;
+ import java.util.concurrent.atomic.AtomicBoolean;
+
+ import org.apache.lucene.document.Document;
+ import org.apache.lucene.document.Field;
+ import org.apache.lucene.document.NumericDocValuesField;
+ import org.apache.lucene.index.DirectoryReader;
+ import org.apache.lucene.index.IndexReader;
+ import org.apache.lucene.index.IndexWriter;
+ import org.apache.lucene.index.IndexWriterConfig;
+ import org.apache.lucene.index.LeafReaderContext;
+ import org.apache.lucene.index.MultiDocValues;
+ import org.apache.lucene.index.NumericDocValues;
+ import org.apache.lucene.index.RandomIndexWriter;
+ import org.apache.lucene.index.Term;
+ import org.apache.lucene.search.IndexSearcher;
+ import org.apache.lucene.search.Query;
+ import org.apache.lucene.search.SimpleCollector;
+ import org.apache.lucene.store.Directory;
+ import org.apache.lucene.store.MockDirectoryWrapper;
+ import org.apache.lucene.util.FixedBitSet;
+ import org.apache.lucene.util.IOUtils;
+ import org.apache.lucene.util.LuceneTestCase;
+ import org.apache.lucene.util.SloppyMath;
+ import org.apache.lucene.util.TestUtil;
+ import org.junit.BeforeClass;
+
+ // TODO: cutover TestGeoUtils too?
+
+ public abstract class BaseGeoPointTestCase extends LuceneTestCase {
+
+ protected static final String FIELD_NAME = "point";
+
+ private static final double LON_SCALE = (0x1L<< GeoEncodingUtils.BITS)/360.0D;
+ private static final double LAT_SCALE = (0x1L<< GeoEncodingUtils.BITS)/180.0D;
+
+ private static double originLat;
+ private static double originLon;
+ private static double lonRange;
+ private static double latRange;
+
+ @BeforeClass
+ public static void beforeClassBase() throws Exception {
+ // Between 1.0 and 3.0:
+ lonRange = 2 * (random().nextDouble() + 0.5);
+ latRange = 2 * (random().nextDouble() + 0.5);
+
+ originLon = GeoUtils.normalizeLon(GeoUtils.MIN_LON_INCL + lonRange + (GeoUtils.MAX_LON_INCL - GeoUtils.MIN_LON_INCL - 2 * lonRange) * random().nextDouble());
+ originLat = GeoUtils.normalizeLat(GeoUtils.MIN_LAT_INCL + latRange + (GeoUtils.MAX_LAT_INCL - GeoUtils.MIN_LAT_INCL - 2 * latRange) * random().nextDouble());
+ }
+
+ /** Return true when testing on a non-small region may be too slow (GeoPoint*Query) */
+ protected boolean forceSmall() {
+ return false;
+ }
+
+ // A particularly tricky adversary for BKD tree:
+ public void testSamePointManyTimes() throws Exception {
+
+ // For GeoPointQuery, only run this test nightly:
+ assumeTrue("GeoPoint*Query is too slow otherwise", TEST_NIGHTLY || forceSmall() == false);
+
+ int numPoints = atLeast(1000);
+ boolean small = random().nextBoolean();
+
+ // Every doc has 2 points:
+ double theLat = randomLat(small);
+ double theLon = randomLon(small);
+
+ double[] lats = new double[numPoints];
+ Arrays.fill(lats, theLat);
+
+ double[] lons = new double[numPoints];
+ Arrays.fill(lons, theLon);
+
+ verify(small, lats, lons);
+ }
+
+ public void testAllLatEqual() throws Exception {
+
+ // For GeoPointQuery, only run this test nightly:
+ assumeTrue("GeoPoint*Query is too slow otherwise", TEST_NIGHTLY || forceSmall() == false);
+
+ int numPoints = atLeast(10000);
+ boolean small = forceSmall() || random().nextBoolean();
+ double lat = randomLat(small);
+ double[] lats = new double[numPoints];
+ double[] lons = new double[numPoints];
+
+ boolean haveRealDoc = false;
+
+ for(int docID=0;docID<numPoints;docID++) {
+ int x = random().nextInt(20);
+ if (x == 17) {
+ // Some docs don't have a point:
+ lats[docID] = Double.NaN;
+ if (VERBOSE) {
+ System.out.println(" doc=" + docID + " is missing");
+ }
+ continue;
+ }
+
+ if (docID > 0 && x == 14 && haveRealDoc) {
+ int oldDocID;
+ while (true) {
+ oldDocID = random().nextInt(docID);
+ if (Double.isNaN(lats[oldDocID]) == false) {
+ break;
+ }
+ }
+
+ // Fully identical point:
+ lons[docID] = lons[oldDocID];
+ if (VERBOSE) {
+ System.out.println(" doc=" + docID + " lat=" + lat + " lon=" + lons[docID] + " (same lat/lon as doc=" + oldDocID + ")");
+ }
+ } else {
+ lons[docID] = randomLon(small);
+ haveRealDoc = true;
+ if (VERBOSE) {
+ System.out.println(" doc=" + docID + " lat=" + lat + " lon=" + lons[docID]);
+ }
+ }
+ lats[docID] = lat;
+ }
+
+ verify(small, lats, lons);
+ }
+
+ public void testAllLonEqual() throws Exception {
+
+ // For GeoPointQuery, only run this test nightly:
+ assumeTrue("GeoPoint*Query is too slow otherwise", TEST_NIGHTLY || forceSmall() == false);
+
+ int numPoints = atLeast(10000);
+ boolean small = forceSmall() || random().nextBoolean();
+ double theLon = randomLon(small);
+ double[] lats = new double[numPoints];
+ double[] lons = new double[numPoints];
+
+ boolean haveRealDoc = false;
+
+ //System.out.println("theLon=" + theLon);
+
+ for(int docID=0;docID<numPoints;docID++) {
+ int x = random().nextInt(20);
+ if (x == 17) {
+ // Some docs don't have a point:
+ lats[docID] = Double.NaN;
+ if (VERBOSE) {
+ System.out.println(" doc=" + docID + " is missing");
+ }
+ continue;
+ }
+
+ if (docID > 0 && x == 14 && haveRealDoc) {
+ int oldDocID;
+ while (true) {
+ oldDocID = random().nextInt(docID);
+ if (Double.isNaN(lats[oldDocID]) == false) {
+ break;
+ }
+ }
+
+ // Fully identical point:
+ lats[docID] = lats[oldDocID];
+ if (VERBOSE) {
+ System.out.println(" doc=" + docID + " lat=" + lats[docID] + " lon=" + theLon + " (same lat/lon as doc=" + oldDocID + ")");
+ }
+ } else {
+ lats[docID] = randomLat(small);
+ haveRealDoc = true;
+ if (VERBOSE) {
+ System.out.println(" doc=" + docID + " lat=" + lats[docID] + " lon=" + theLon);
+ }
+ }
+ lons[docID] = theLon;
+ }
+
+ verify(small, lats, lons);
+ }
+
+ public void testMultiValued() throws Exception {
+
+ // For GeoPointQuery, only run this test nightly:
+ assumeTrue("GeoPoint*Query is too slow otherwise", TEST_NIGHTLY || forceSmall() == false);
+
+ int numPoints = atLeast(10000);
+ // Every doc has 2 points:
+ double[] lats = new double[2*numPoints];
+ double[] lons = new double[2*numPoints];
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig();
+ initIndexWriterConfig(FIELD_NAME, iwc);
+
+ // We rely on docID order:
+ iwc.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+
+ boolean small = random().nextBoolean();
+
+ for (int id=0;id<numPoints;id++) {
+ Document doc = new Document();
+ lats[2*id] = randomLat(small);
+ lons[2*id] = randomLon(small);
+ doc.add(newStringField("id", ""+id, Field.Store.YES));
+ addPointToDoc(FIELD_NAME, doc, lats[2*id], lons[2*id]);
+ lats[2*id+1] = randomLat(small);
+ lons[2*id+1] = randomLon(small);
+ addPointToDoc(FIELD_NAME, doc, lats[2*id+1], lons[2*id+1]);
+
+ if (VERBOSE) {
+ System.out.println("id=" + id);
+ System.out.println(" lat=" + lats[2*id] + " lon=" + lons[2*id]);
+ System.out.println(" lat=" + lats[2*id+1] + " lon=" + lons[2*id+1]);
+ }
+ w.addDocument(doc);
+ }
+
+ // TODO: share w/ verify; just need parallel array of the expected ids
+ if (random().nextBoolean()) {
+ w.forceMerge(1);
+ }
+ IndexReader r = w.getReader();
+ w.close();
+
+ // We can't wrap with "exotic" readers because the BKD query must see the BKDDVFormat:
+ IndexSearcher s = newSearcher(r, false);
+
+ int iters = atLeast(75);
+ for (int iter=0;iter<iters;iter++) {
+ GeoRect rect = randomRect(small, small == false);
+
+ if (VERBOSE) {
+ System.out.println("\nTEST: iter=" + iter + " rect=" + rect);
+ }
+
+ Query query = newRectQuery(FIELD_NAME, rect);
+
+ final FixedBitSet hits = new FixedBitSet(r.maxDoc());
+ s.search(query, new SimpleCollector() {
+
+ private int docBase;
+
+ @Override
+ public boolean needsScores() {
+ return false;
+ }
+
+ @Override
+ protected void doSetNextReader(LeafReaderContext context) throws IOException {
+ docBase = context.docBase;
+ }
+
+ @Override
+ public void collect(int doc) {
+ hits.set(docBase+doc);
+ }
+ });
+
+ boolean fail = false;
+
+ for(int docID=0;docID<lats.length/2;docID++) {
+ double latDoc1 = lats[2*docID];
+ double lonDoc1 = lons[2*docID];
+ double latDoc2 = lats[2*docID+1];
+ double lonDoc2 = lons[2*docID+1];
+
+ Boolean result1 = rectContainsPoint(rect, latDoc1, lonDoc1);
+ if (result1 == null) {
+ // borderline case: cannot test
+ continue;
+ }
+
+ Boolean result2 = rectContainsPoint(rect, latDoc2, lonDoc2);
+ if (result2 == null) {
+ // borderline case: cannot test
+ continue;
+ }
+
+ boolean expected = result1 == Boolean.TRUE || result2 == Boolean.TRUE;
+
+ if (hits.get(docID) != expected) {
+ String id = s.doc(docID).get("id");
+ if (expected) {
+ System.out.println(Thread.currentThread().getName() + ": id=" + id + " docID=" + docID + " should match but did not");
+ } else {
+ System.out.println(Thread.currentThread().getName() + ": id=" + id + " docID=" + docID + " should not match but did");
+ }
+ System.out.println(" rect=" + rect);
+ System.out.println(" lat=" + latDoc1 + " lon=" + lonDoc1 + "\n lat=" + latDoc2 + " lon=" + lonDoc2);
+ System.out.println(" result1=" + result1 + " result2=" + result2);
+ fail = true;
+ }
+ }
+
+ if (fail) {
+ fail("some hits were wrong");
+ }
+ }
+ r.close();
+ dir.close();
+ }
+
+ public void testRandomTiny() throws Exception {
+ // Make sure single-leaf-node case is OK:
+ doTestRandom(10);
+ }
+
+ public void testRandomMedium() throws Exception {
+ doTestRandom(10000);
+ }
+
+ @Nightly
+ public void testRandomBig() throws Exception {
+ assumeFalse("Direct codec can OOME on this test", TestUtil.getDocValuesFormat(FIELD_NAME).equals("Direct"));
+ assumeFalse("Memory codec can OOME on this test", TestUtil.getDocValuesFormat(FIELD_NAME).equals("Memory"));
+ doTestRandom(200000);
+ }
+
+ private void doTestRandom(int count) throws Exception {
+
+ int numPoints = atLeast(count);
+
+ if (VERBOSE) {
+ System.out.println("TEST: numPoints=" + numPoints);
+ }
+
+ double[] lats = new double[numPoints];
+ double[] lons = new double[numPoints];
+
+ boolean small = random().nextBoolean();
+
+ boolean haveRealDoc = false;
+
+ for (int id=0;id<numPoints;id++) {
+ int x = random().nextInt(20);
+ if (x == 17) {
+ // Some docs don't have a point:
+ lats[id] = Double.NaN;
+ if (VERBOSE) {
+ System.out.println(" id=" + id + " is missing");
+ }
+ continue;
+ }
+
+ if (id > 0 && x < 3 && haveRealDoc) {
+ int oldID;
+ while (true) {
+ oldID = random().nextInt(id);
+ if (Double.isNaN(lats[oldID]) == false) {
+ break;
+ }
+ }
+
+ if (x == 0) {
+ // Identical lat to old point
+ lats[id] = lats[oldID];
+ lons[id] = randomLon(small);
+ if (VERBOSE) {
+ System.out.println(" id=" + id + " lat=" + lats[id] + " lon=" + lons[id] + " (same lat as doc=" + oldID + ")");
+ }
+ } else if (x == 1) {
+ // Identical lon to old point
+ lats[id] = randomLat(small);
+ lons[id] = lons[oldID];
+ if (VERBOSE) {
+ System.out.println(" id=" + id + " lat=" + lats[id] + " lon=" + lons[id] + " (same lon as doc=" + oldID + ")");
+ }
+ } else {
+ assert x == 2;
+ // Fully identical point:
+ lats[id] = lats[oldID];
+ lons[id] = lons[oldID];
+ if (VERBOSE) {
+ System.out.println(" id=" + id + " lat=" + lats[id] + " lon=" + lons[id] + " (same lat/lon as doc=" + oldID + ")");
+ }
+ }
+ } else {
+ lats[id] = randomLat(small);
+ lons[id] = randomLon(small);
+ haveRealDoc = true;
+ if (VERBOSE) {
+ System.out.println(" id=" + id + " lat=" + lats[id] + " lon=" + lons[id]);
+ }
+ }
+ }
+
+ verify(small, lats, lons);
+ }
+
+ public double randomLat(boolean small) {
+ double result;
+ if (small) {
+ result = GeoUtils.normalizeLat(originLat + latRange * (random().nextDouble() - 0.5));
+ } else {
+ result = -90 + 180.0 * random().nextDouble();
+ }
+ return result;
+ }
+
+ public double randomLon(boolean small) {
+ double result;
+ if (small) {
+ result = GeoUtils.normalizeLon(originLon + lonRange * (random().nextDouble() - 0.5));
+ } else {
+ result = -180 + 360.0 * random().nextDouble();
+ }
+ return result;
+ }
+
+ protected GeoRect randomRect(boolean small, boolean canCrossDateLine) {
+ double lat0 = randomLat(small);
+ double lat1 = randomLat(small);
+ double lon0 = randomLon(small);
+ double lon1 = randomLon(small);
+
+ if (lat1 < lat0) {
+ double x = lat0;
+ lat0 = lat1;
+ lat1 = x;
+ }
+
+ if (canCrossDateLine == false && lon1 < lon0) {
+ double x = lon0;
+ lon0 = lon1;
+ lon1 = x;
+ }
+
+ return new GeoRect(lon0, lon1, lat0, lat1);
+ }
+
+ protected void initIndexWriterConfig(String field, IndexWriterConfig iwc) {
+ }
+
+ protected abstract void addPointToDoc(String field, Document doc, double lat, double lon);
+
+ protected abstract Query newRectQuery(String field, GeoRect bbox);
+
+ protected abstract Query newDistanceQuery(String field, double centerLat, double centerLon, double radiusMeters);
+
+ protected abstract Query newDistanceRangeQuery(String field, double centerLat, double centerLon, double minRadiusMeters, double radiusMeters);
+
+ protected abstract Query newPolygonQuery(String field, double[] lats, double[] lons);
+
+ /** Returns null if it's borderline case */
+ protected abstract Boolean rectContainsPoint(GeoRect rect, double pointLat, double pointLon);
+
+ /** Returns null if it's borderline case */
+ protected abstract Boolean polyRectContainsPoint(GeoRect rect, double pointLat, double pointLon);
+
+ /** Returns null if it's borderline case */
+ protected abstract Boolean circleContainsPoint(double centerLat, double centerLon, double radiusMeters, double pointLat, double pointLon);
+
+ protected abstract Boolean distanceRangeContainsPoint(double centerLat, double centerLon, double minRadiusMeters, double radiusMeters, double pointLat, double pointLon);
+
+ private static abstract class VerifyHits {
+
+ public void test(AtomicBoolean failed, boolean small, IndexSearcher s, NumericDocValues docIDToID, Set<Integer> deleted, Query query, double[] lats, double[] lons) throws Exception {
+ int maxDoc = s.getIndexReader().maxDoc();
+ final FixedBitSet hits = new FixedBitSet(maxDoc);
+ s.search(query, new SimpleCollector() {
+
+ private int docBase;
+
+ @Override
+ public boolean needsScores() {
+ return false;
+ }
+
+ @Override
+ protected void doSetNextReader(LeafReaderContext context) throws IOException {
+ docBase = context.docBase;
+ }
+
+ @Override
+ public void collect(int doc) {
+ hits.set(docBase+doc);
+ }
+ });
+
+ boolean fail = false;
+
+ for(int docID=0;docID<maxDoc;docID++) {
+ int id = (int) docIDToID.get(docID);
+ Boolean expected;
+ if (deleted.contains(id)) {
+ expected = false;
+ } else if (Double.isNaN(lats[id])) {
+ expected = false;
+ } else {
+ expected = shouldMatch(lats[id], lons[id]);
+ }
+
+ // null means it's a borderline case which is allowed to be wrong:
+ if (expected != null && hits.get(docID) != expected) {
+ if (expected) {
+ System.out.println(Thread.currentThread().getName() + ": id=" + id + " should match but did not");
+ } else {
+ System.out.println(Thread.currentThread().getName() + ": id=" + id + " should not match but did");
+ }
+ System.out.println(" small=" + small + " query=" + query +
+ " docID=" + docID + "\n lat=" + lats[id] + " lon=" + lons[id] +
+ "\n deleted?=" + deleted.contains(id));
+ if (Double.isNaN(lats[id]) == false) {
+ describe(docID, lats[id], lons[id]);
+ }
+ fail = true;
+ }
+ }
+
+ if (fail) {
+ failed.set(true);
+ fail("some hits were wrong");
+ }
+ }
+
+ /** Return true if we definitely should match, false if we definitely
+ * should not match, and null if it's a borderline case which might
+ * go either way. */
+ protected abstract Boolean shouldMatch(double lat, double lon);
+
+ protected abstract void describe(int docID, double lat, double lon);
+ }
+
+ protected void verify(boolean small, double[] lats, double[] lons) throws Exception {
+ IndexWriterConfig iwc = newIndexWriterConfig();
+ // Else we can get O(N^2) merging:
+ int mbd = iwc.getMaxBufferedDocs();
+ if (mbd != -1 && mbd < lats.length/100) {
+ iwc.setMaxBufferedDocs(lats.length/100);
+ }
+ Directory dir;
+ if (lats.length > 100000) {
+ dir = newFSDirectory(createTempDir(getClass().getSimpleName()));
+ } else {
+ dir = newDirectory();
+ }
+
+ Set<Integer> deleted = new HashSet<>();
+ // RandomIndexWriter is too slow here:
+ IndexWriter w = new IndexWriter(dir, iwc);
+ for(int id=0;id<lats.length;id++) {
+ Document doc = new Document();
+ doc.add(newStringField("id", ""+id, Field.Store.NO));
+ doc.add(new NumericDocValuesField("id", id));
+ if (Double.isNaN(lats[id]) == false) {
+ addPointToDoc(FIELD_NAME, doc, lats[id], lons[id]);
+ }
+ w.addDocument(doc);
+ if (id > 0 && random().nextInt(100) == 42) {
+ int idToDelete = random().nextInt(id);
+ w.deleteDocuments(new Term("id", ""+idToDelete));
+ deleted.add(idToDelete);
+ if (VERBOSE) {
+ System.out.println(" delete id=" + idToDelete);
+ }
+ }
+ }
+
+ if (random().nextBoolean()) {
+ w.forceMerge(1);
+ }
+ final IndexReader r = DirectoryReader.open(w);
+ w.close();
+
+ // We can't wrap with "exotic" readers because the BKD query must see the BKDDVFormat:
+ IndexSearcher s = newSearcher(r, false);
+
+ // Make sure queries are thread safe:
+ int numThreads = TestUtil.nextInt(random(), 2, 5);
+
+ List<Thread> threads = new ArrayList<>();
+ final int iters = atLeast(75);
+
+ final CountDownLatch startingGun = new CountDownLatch(1);
+ final AtomicBoolean failed = new AtomicBoolean();
+
+ for(int i=0;i<numThreads;i++) {
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ _run();
+ } catch (Exception e) {
+ failed.set(true);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void _run() throws Exception {
+ startingGun.await();
+
+ NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
+
+ for (int iter=0;iter<iters && failed.get() == false;iter++) {
+
+ if (VERBOSE) {
- System.out.println("\nTEST: iter=" + iter + " s=" + s);
++ System.out.println("\n" + Thread.currentThread().getName() + ": TEST: iter=" + iter + " s=" + s);
+ }
+ Query query;
+ VerifyHits verifyHits;
+
+ if (random().nextBoolean()) {
+ // Rect: don't allow dateline crossing when testing small:
+ final GeoRect rect = randomRect(small, small == false);
+
+ query = newRectQuery(FIELD_NAME, rect);
+
+ verifyHits = new VerifyHits() {
+ @Override
+ protected Boolean shouldMatch(double pointLat, double pointLon) {
+ return rectContainsPoint(rect, pointLat, pointLon);
+ }
+ @Override
+ protected void describe(int docID, double lat, double lon) {
+ }
+ };
+
+ } else if (random().nextBoolean()) {
+ // Distance
+ final boolean rangeQuery = random().nextBoolean();
+ final double centerLat = randomLat(small);
+ final double centerLon = randomLon(small);
+
+ double radiusMeters;
+ double minRadiusMeters;
+
+ if (small) {
+ // Approx 3 degrees lon at the equator:
+ radiusMeters = random().nextDouble() * 333000 + 1.0;
+ } else {
+ // So the query can cover at most 50% of the earth's surface:
+ radiusMeters = random().nextDouble() * GeoProjectionUtils.SEMIMAJOR_AXIS * Math.PI / 2.0 + 1.0;
+ }
+
+ // generate a random minimum radius between 1% and 95% the max radius
+ minRadiusMeters = (0.01 + 0.94 * random().nextDouble()) * radiusMeters;
+
+ if (VERBOSE) {
+ final DecimalFormat df = new DecimalFormat("#,###.00", DecimalFormatSymbols.getInstance(Locale.ENGLISH));
+ System.out.println(" radiusMeters = " + df.format(radiusMeters)
+ + ((rangeQuery == true) ? " minRadiusMeters = " + df.format(minRadiusMeters) : ""));
+ }
+
+ try {
+ if (rangeQuery == true) {
+ query = newDistanceRangeQuery(FIELD_NAME, centerLat, centerLon, minRadiusMeters, radiusMeters);
+ } else {
+ query = newDistanceQuery(FIELD_NAME, centerLat, centerLon, radiusMeters);
+ }
+ } catch (IllegalArgumentException e) {
+ if (e.getMessage().contains("exceeds maxRadius")) {
+ continue;
+ }
+ throw e;
+ }
+
+ verifyHits = new VerifyHits() {
+ @Override
+ protected Boolean shouldMatch(double pointLat, double pointLon) {
+ if (rangeQuery == false) {
+ return circleContainsPoint(centerLat, centerLon, radiusMeters, pointLat, pointLon);
+ } else {
+ return distanceRangeContainsPoint(centerLat, centerLon, minRadiusMeters, radiusMeters, pointLat, pointLon);
+ }
+ }
+
+ @Override
+ protected void describe(int docID, double pointLat, double pointLon) {
+ double distanceKM = SloppyMath.haversin(centerLat, centerLon, pointLat, pointLon);
+ System.out.println(" docID=" + docID + " centerLon=" + centerLon + " centerLat=" + centerLat
+ + " pointLon=" + pointLon + " pointLat=" + pointLat + " distanceMeters=" + (distanceKM * 1000)
+ + " vs" + ((rangeQuery == true) ? " minRadiusMeters=" + minRadiusMeters : "") + " radiusMeters=" + radiusMeters);
+ }
+ };
+
+ // TODO: get poly query working with dateline crossing too (how?)!
+ } else {
+
+ // TODO: poly query can't handle dateline crossing yet:
+ final GeoRect bbox = randomRect(small, false);
+
+ // Polygon
+ double[] lats = new double[5];
+ double[] lons = new double[5];
+ lats[0] = bbox.minLat;
+ lons[0] = bbox.minLon;
+ lats[1] = bbox.maxLat;
+ lons[1] = bbox.minLon;
+ lats[2] = bbox.maxLat;
+ lons[2] = bbox.maxLon;
+ lats[3] = bbox.minLat;
+ lons[3] = bbox.maxLon;
+ lats[4] = bbox.minLat;
+ lons[4] = bbox.minLon;
+ query = newPolygonQuery(FIELD_NAME, lats, lons);
+
+ verifyHits = new VerifyHits() {
+ @Override
+ protected Boolean shouldMatch(double pointLat, double pointLon) {
+ return polyRectContainsPoint(bbox, pointLat, pointLon);
+ }
+
+ @Override
+ protected void describe(int docID, double lat, double lon) {
+ }
+ };
+ }
+
+ if (query != null) {
+
+ if (VERBOSE) {
+ System.out.println(" query=" + query);
+ }
+
+ verifyHits.test(failed, small, s, docIDToID, deleted, query, lats, lons);
+ }
+ }
+ }
+ };
+ thread.setName("T" + i);
+ thread.start();
+ threads.add(thread);
+ }
+ startingGun.countDown();
+ for(Thread thread : threads) {
+ thread.join();
+ }
+ IOUtils.close(r, dir);
+ assertFalse(failed.get());
+ }
+ }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd6804bc/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd6804bc/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd6804bc/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
----------------------------------------------------------------------
diff --cc lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
index b19045b,51dbffe..a5fc397
--- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
@@@ -45,7 -43,8 +45,9 @@@ import org.apache.lucene.index.IndexFil
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.NoDeletionPolicy;
+import org.apache.lucene.index.SegmentInfos;
+ import org.apache.lucene.mockfile.FilterFileSystem;
+ import org.apache.lucene.mockfile.VirusCheckingFS;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@@ -239,24 -219,8 +222,16 @@@ public class MockDirectoryWrapper exten
throw new IOException("cannot rename after crash");
}
- if (openFiles.containsKey(source)) {
- if (assertNoDeleteOpenFile) {
- throw (AssertionError) fillOpenTrace(new AssertionError("MockDirectoryWrapper: source file \"" + source + "\" is still open: cannot rename"), source, true);
- } else if (noDeleteOpenFile) {
- throw (IOException) fillOpenTrace(new IOException("MockDirectoryWrapper: source file \"" + source + "\" is still open: cannot rename"), source, true);
- }
- if (assertNoDeleteOpenFile && openFiles.containsKey(source)) {
- throw (AssertionError) fillOpenTrace(new AssertionError("MockDirectoryWrapper: file \"" + source + "\" is still open: cannot rename"), source, true);
++ if (openFiles.containsKey(source) && assertNoDeleteOpenFile) {
++ throw (AssertionError) fillOpenTrace(new AssertionError("MockDirectoryWrapper: source file \"" + source + "\" is still open: cannot rename"), source, true);
+ }
+
- if (openFiles.containsKey(dest)) {
- if (assertNoDeleteOpenFile) {
- throw (AssertionError) fillOpenTrace(new AssertionError("MockDirectoryWrapper: dest file \"" + dest + "\" is still open: cannot rename"), dest, true);
- } else if (noDeleteOpenFile) {
- throw (IOException) fillOpenTrace(new IOException("MockDirectoryWrapper: dest file \"" + dest + "\" is still open: cannot rename"), dest, true);
- }
++ if (openFiles.containsKey(dest) && assertNoDeleteOpenFile) {
++ throw (AssertionError) fillOpenTrace(new AssertionError("MockDirectoryWrapper: dest file \"" + dest + "\" is still open: cannot rename"), dest, true);
+ }
+
+ if (createdFiles.contains(dest)) {
+ throw new IOException("MockDirectoryWrapper: dest file \"" + dest + "\" already exists: cannot rename");
}
boolean success = false;
@@@ -271,8 -235,6 +246,7 @@@
unSyncedFiles.add(dest);
}
openFilesDeleted.remove(source);
- triedToDelete.remove(dest);
+ createdFiles.add(dest);
}
}
}
@@@ -294,213 -256,94 +268,190 @@@
}
}
- /** Simulates a crash of OS or machine by overwriting
- * unsynced files. */
- public synchronized void crash() throws IOException {
- openFiles = new HashMap<>();
- openFilesForWrite = new HashSet<>();
- openFilesDeleted = new HashSet<>();
- Iterator<String> it = unSyncedFiles.iterator();
- unSyncedFiles = new HashSet<>();
- // first force-close all files, so we can corrupt on windows etc.
- // clone the file map, as these guys want to remove themselves on close.
- Map<Closeable,Exception> m = new IdentityHashMap<>(openFileHandles);
- for (Closeable f : m.keySet()) {
- try {
- f.close();
- } catch (Exception ignored) {}
+ public synchronized void corruptUnknownFiles() throws IOException {
+
+ System.out.println("MDW: corrupt unknown files");
+ Set<String> knownFiles = new HashSet<>();
+ for(String fileName : listAll()) {
+ if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
+ System.out.println("MDW: read " + fileName + " to gather files it references");
+ knownFiles.addAll(SegmentInfos.readCommit(this, fileName).files(true));
+ }
}
- // Maybe disable virus checker so it doesn't interfere with our efforts to corrupt files below:
- boolean virusCheckerWasEnabled = TestUtil.disableVirusChecker(in);
+ Set<String> toCorrupt = new HashSet<>();
+ Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
+ for(String fileName : listAll()) {
+ m.reset(fileName);
+ if (knownFiles.contains(fileName) == false &&
+ fileName.endsWith("write.lock") == false &&
+ (m.matches() || fileName.startsWith(IndexFileNames.PENDING_SEGMENTS))) {
+ toCorrupt.add(fileName);
+ }
+ }
- while(it.hasNext()) {
- String name = it.next();
- int damage = randomState.nextInt(5);
+ corruptFiles(toCorrupt);
+ }
+
- public synchronized void corruptFiles(Collection<String> files) {
++ public synchronized void corruptFiles(Collection<String> files) throws IOException {
+ // Must make a copy because we change the incoming unsyncedFiles
+ // when we create temp files, delete, etc., below:
+ for(String name : new ArrayList<>(files)) {
+ int damage = randomState.nextInt(6);
String action = null;
- if (damage == 0) {
+ switch(damage) {
+
+ case 0:
action = "deleted";
- try {
- deleteFile(name, true);
- } catch (IOException ioe) {
- // ignore
- }
+ deleteFile(name);
- } else if (damage == 1) {
+ break;
+
+ case 1:
action = "zeroed";
// Zero out file entirely
- long length = fileLength(name);
+ long length;
+ try {
+ length = fileLength(name);
+ } catch (IOException ioe) {
+ // Ignore
+ continue;
+ }
byte[] zeroes = new byte[256];
long upto = 0;
- IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState));
- while(upto < length) {
- final int limit = (int) Math.min(length-upto, zeroes.length);
- out.writeBytes(zeroes, 0, limit);
- upto += limit;
- }
- out.close();
- } else if (damage == 2) {
- action = "partially truncated";
- // Partially Truncate the file:
-
- // First, make temp file and copy only half this
- // file over:
- String tempFileName;
- while (true) {
- tempFileName = ""+randomState.nextInt();
- if (!LuceneTestCase.slowFileExists(in, tempFileName)) {
- break;
+ try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) {
+ while(upto < length) {
+ final int limit = (int) Math.min(length-upto, zeroes.length);
+ out.writeBytes(zeroes, 0, limit);
+ upto += limit;
}
+ } catch (IOException ioe) {
+ // ignore
}
- final IndexOutput tempOut = in.createOutput(tempFileName, LuceneTestCase.newIOContext(randomState));
- IndexInput ii = in.openInput(name, LuceneTestCase.newIOContext(randomState));
- tempOut.copyBytes(ii, ii.length()/2);
- tempOut.close();
- ii.close();
+ break;
- // Delete original and copy bytes back:
- deleteFile(name);
-
- try(IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) {
- ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState));
- out.copyBytes(ii, ii.length());
- ii.close();
+ case 2:
+ {
+ action = "partially truncated";
+ // Partially Truncate the file:
+
+ // First, make temp file and copy only half this
+ // file over:
+ String tempFileName = null;
+ try (IndexOutput tempOut = in.createTempOutput("name", "mdw_corrupt", LuceneTestCase.newIOContext(randomState));
+ IndexInput ii = in.openInput(name, LuceneTestCase.newIOContext(randomState))) {
+ tempFileName = tempOut.getName();
+ tempOut.copyBytes(ii, ii.length()/2);
+ } catch (IOException ioe) {
+ // ignore
+ }
+
- try {
- // Delete original and copy bytes back:
- deleteFile(name, true);
- } catch (IOException ioe) {
- // ignore
- }
++ // Delete original and copy bytes back:
++ deleteFile(name);
+
+ try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState));
+ IndexInput ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState))) {
+ out.copyBytes(ii, ii.length());
+ } catch (IOException ioe) {
+ // ignore
+ }
- try {
- deleteFile(tempFileName, true);
- } catch (IOException ioe) {
- // ignore
- }
++ deleteFile(tempFileName);
}
- deleteFile(tempFileName);
- } else if (damage == 3) {
+ break;
+
+ case 3:
// The file survived intact:
action = "didn't change";
- } else {
+ break;
+
+ case 4:
+ // Corrupt one bit randomly in the file:
+
+ {
+
+ String tempFileName = null;
+ try (IndexOutput tempOut = in.createTempOutput("name", "mdw_corrupt", LuceneTestCase.newIOContext(randomState));
+ IndexInput ii = in.openInput(name, LuceneTestCase.newIOContext(randomState))) {
+ tempFileName = tempOut.getName();
+ if (ii.length() > 0) {
+ // Copy first part unchanged:
+ long byteToCorrupt = (long) (randomState.nextDouble() * ii.length());
+ if (byteToCorrupt > 0) {
+ tempOut.copyBytes(ii, byteToCorrupt);
+ }
+
+ // Randomly flip one bit from this byte:
+ byte b = ii.readByte();
+ int bitToFlip = randomState.nextInt(8);
+ b = (byte) (b ^ (1 << bitToFlip));
+ tempOut.writeByte(b);
+
+ action = "flip bit " + bitToFlip + " of byte " + byteToCorrupt + " out of " + ii.length() + " bytes";
+
+ // Copy last part unchanged:
+ long bytesLeft = ii.length() - byteToCorrupt - 1;
+ if (bytesLeft > 0) {
+ tempOut.copyBytes(ii, bytesLeft);
+ }
+ } else {
+ action = "didn't change";
+ }
+ } catch (IOException ioe) {
+ // ignore
+ }
+
- try {
- // Delete original and copy bytes back:
- deleteFile(name, true);
- } catch (IOException ioe) {
- // ignore
- }
++ // Delete original and copy bytes back:
++ deleteFile(name);
+
+ try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState));
+ IndexInput ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState))) {
+ out.copyBytes(ii, ii.length());
+ } catch (IOException ioe) {
+ // ignore
+ }
- try {
- deleteFile(tempFileName, true);
- } catch (IOException ioe) {
- // ignore
- }
++
++ deleteFile(tempFileName);
+ }
+ break;
+
+ case 5:
action = "fully truncated";
// Totally truncate the file to zero bytes
- try {
- deleteFile(name, true);
- } catch (IOException ioe) {
- // ignore
- }
+ deleteFile(name);
+
try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) {
+ } catch (IOException ioe) {
+ // ignore
}
+ break;
+
+ default:
+ throw new AssertionError();
}
- // Re-enable
- if (virusCheckerWasEnabled) {
- TestUtil.enableVirusChecker(in);
- }
- if (LuceneTestCase.VERBOSE) {
+
+ if (true || LuceneTestCase.VERBOSE) {
System.out.println("MockDirectoryWrapper: " + action + " unsynced file: " + name);
}
}
+ }
+
+ /** Simulates a crash of OS or machine by overwriting
+ * unsynced files. */
- public synchronized void crash() {
++ public synchronized void crash() throws IOException {
crashed = true;
+ openFiles = new HashMap<>();
+ openFilesForWrite = new HashSet<>();
+ openFilesDeleted = new HashSet<>();
+ // first force-close all files, so we can corrupt on windows etc.
+ // clone the file map, as these guys want to remove themselves on close.
+ Map<Closeable,Exception> m = new IdentityHashMap<>(openFileHandles);
+ for (Closeable f : m.keySet()) {
+ try {
+ f.close();
+ } catch (Exception ignored) {}
+ }
+ corruptFiles(unSyncedFiles);
+ unSyncedFiles = new HashSet<>();
}
public synchronized void clearCrash() {
@@@ -942,13 -745,12 +853,14 @@@
String[] startFiles = allFiles.toArray(new String[0]);
IndexWriterConfig iwc = new IndexWriterConfig(null);
iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
+
+ // We must do this before opening writer otherwise writer will be angry if there are pending deletions:
+ TestUtil.disableVirusChecker(in);
+
new IndexWriter(in, iwc).rollback();
- String[] endFiles = in.listAll();
+
+ Set<String> files = new HashSet<>(Arrays.asList(listAll()));
- // Disregard what happens with the pendingDeletions files:
- files.removeAll(pendingDeletions);
+ String[] endFiles = files.toArray(new String[0]);
Set<String> startSet = new TreeSet<>(Arrays.asList(startFiles));
Set<String> endSet = new TreeSet<>(Arrays.asList(endFiles));
@@@ -1027,11 -784,7 +894,7 @@@
extras += "\n\nThese files were added (waaaaaaaaaat!): " + added;
}
- if (pendingDeletions.size() != 0) {
- extras += "\n\nThese files we had previously tried to delete, but couldn't: " + pendingDeletions;
- }
-
- throw new RuntimeException("unreferenced files: before delete:\n " + Arrays.toString(startFiles) + "\n after delete:\n " + Arrays.toString(endFiles) + extras);
+ throw new RuntimeException(this + ": unreferenced files: before delete:\n " + Arrays.toString(startFiles) + "\n after delete:\n " + Arrays.toString(endFiles) + extras);
}
DirectoryReader ir1 = DirectoryReader.open(this);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd6804bc/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd6804bc/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
----------------------------------------------------------------------