You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/11/16 16:55:57 UTC
lucene-solr:master: add basic sorted index bwc test
Repository: lucene-solr
Updated Branches:
refs/heads/master 53a0748f4 -> 774e31b6d
add basic sorted index bwc test
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/774e31b6
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/774e31b6
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/774e31b6
Branch: refs/heads/master
Commit: 774e31b6dd7184fb6d43fca83e32fcb46da32e20
Parents: 53a0748
Author: Mike McCandless <mi...@apache.org>
Authored: Wed Nov 16 11:54:22 2016 -0500
Committer: Mike McCandless <mi...@apache.org>
Committed: Wed Nov 16 11:55:29 2016 -0500
----------------------------------------------------------------------
.../index/TestBackwardsCompatibility.java | 82 +++++++++++++++++++
.../org/apache/lucene/index/sorted.6.2.0.zip | Bin 0 -> 157007 bytes
.../org/apache/lucene/index/sorted.6.2.1.zip | Bin 0 -> 100974 bytes
.../org/apache/lucene/index/sorted.6.3.0.zip | Bin 0 -> 71090 bytes
4 files changed, 82 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/774e31b6/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index d77f84b..d924238 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -25,13 +25,18 @@ import java.lang.reflect.Modifier;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.text.ParsePosition;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Random;
+import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -62,6 +67,8 @@ import org.apache.lucene.legacy.LegacyNumericUtils;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory;
@@ -165,6 +172,57 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
// a test option to not remove temp dir...):
Thread.sleep(100000);
}
+
+ // ant test -Dtestcase=TestBackwardsCompatibility -Dtestmethod=testCreateSortedIndex -Dtests.codec=default -Dtests.useSecurityManager=false -Dtests.bwcdir=/tmp/sorted
+ public void testCreateSortedIndex() throws Exception {
+
+ Path indexDir = getIndexDir().resolve("sorted");
+ Files.deleteIfExists(indexDir);
+ Directory dir = newFSDirectory(indexDir);
+
+ LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();
+ mp.setNoCFSRatio(1.0);
+ mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
+ MockAnalyzer analyzer = new MockAnalyzer(random());
+ analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
+
+ // TODO: remove randomness
+ IndexWriterConfig conf = new IndexWriterConfig(analyzer);
+ conf.setMergePolicy(mp);
+ conf.setUseCompoundFile(false);
+ conf.setIndexSort(new Sort(new SortField("dateDV", SortField.Type.LONG, true)));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ LineFileDocs docs = new LineFileDocs(random());
+ SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd", Locale.ROOT);
+ parser.setTimeZone(TimeZone.getTimeZone("UTC"));
+ ParsePosition position = new ParsePosition(0);
+ Field dateDVField = null;
+ for(int i=0;i<50;i++) {
+ Document doc = docs.nextDoc();
+ String dateString = doc.get("date");
+
+ position.setIndex(0);
+ Date date = parser.parse(dateString, position);
+ if (position.getErrorIndex() != -1) {
+ throw new AssertionError("failed to parse \"" + dateString + "\" as date");
+ }
+ if (position.getIndex() != dateString.length()) {
+ throw new AssertionError("failed to parse \"" + dateString + "\" as date");
+ }
+ if (dateDVField == null) {
+ dateDVField = new NumericDocValuesField("dateDV", 0l);
+ doc.add(dateDVField);
+ }
+ dateDVField.setLongValue(date.getTime());
+ if (i == 250) {
+ writer.commit();
+ }
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+ writer.close();
+ dir.close();
+ }
private void updateNumeric(IndexWriter writer, String id, String f, String cf, long value) throws IOException {
writer.updateNumericDocValue(new Term("id", id), f, value);
@@ -1483,6 +1541,30 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
dir.close();
}
}
+
+ public void testSortedIndex() throws Exception {
+ String[] versions = new String[] {"6.2.0", "6.2.1", "6.3.0"};
+ for(String version : versions) {
+ Path path = createTempDir("sorted");
+ InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream("sorted." + version + ".zip");
+ assertNotNull("Sorted index index " + version + " not found", resource);
+ TestUtil.unzip(resource, path);
+
+ // TODO: more tests
+ Directory dir = newFSDirectory(path);
+
+ DirectoryReader reader = DirectoryReader.open(dir);
+ assertEquals(1, reader.leaves().size());
+ Sort sort = reader.leaves().get(0).reader().getIndexSort();
+ assertNotNull(sort);
+ assertEquals("<long: \"dateDV\">!", sort.toString());
+ reader.close();
+
+ // this will confirm the docs really are sorted:
+ TestUtil.checkIndex(dir);
+ dir.close();
+ }
+ }
static long getValue(BinaryDocValues bdv) throws IOException {
BytesRef term = bdv.binaryValue();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/774e31b6/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.2.0.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.2.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.2.0.zip
new file mode 100644
index 0000000..b595d9b
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.2.0.zip differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/774e31b6/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.2.1.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.2.1.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.2.1.zip
new file mode 100644
index 0000000..40d1ccc
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.2.1.zip differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/774e31b6/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.3.0.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.3.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.3.0.zip
new file mode 100644
index 0000000..6d89d781
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.3.0.zip differ