You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/06/16 21:26:23 UTC
svn commit: r1602968 [4/5] - in /lucene/dev/branches/lucene5752: ./
dev-tools/ dev-tools/idea/.idea/ lucene/ lucene/analysis/
lucene/analysis/common/
lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/
lucene/codecs/ lucene/codecs/src/...
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestDocIdSet.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestDocIdSet.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestDocIdSet.java Mon Jun 16 19:26:19 2014
@@ -38,6 +38,11 @@ public class TestDocIdSet extends Lucene
final int maxdoc=10;
final DocIdSet innerSet = new DocIdSet() {
+ @Override
+ public long ramBytesUsed() {
+ return 0L;
+ }
+
@Override
public DocIdSetIterator iterator() {
return new DocIdSetIterator() {
@@ -151,6 +156,11 @@ public class TestDocIdSet extends Lucene
public DocIdSetIterator iterator() {
return null;
}
+
+ @Override
+ public long ramBytesUsed() {
+ return 0L;
+ }
};
return new FilteredDocIdSet(innerNullIteratorSet) {
@Override
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java Mon Jun 16 19:26:19 2014
@@ -435,7 +435,12 @@ public class TestFilteredQuery extends L
bitSet.set(d, true);
}
return new DocIdSet() {
-
+
+ @Override
+ public long ramBytesUsed() {
+ return 0L;
+ }
+
@Override
public Bits bits() throws IOException {
if (nullBitset) {
@@ -502,7 +507,12 @@ public class TestFilteredQuery extends L
public DocIdSet getDocIdSet(final AtomicReaderContext context, Bits acceptDocs)
throws IOException {
return new DocIdSet() {
-
+
+ @Override
+ public long ramBytesUsed() {
+ return 0L;
+ }
+
@Override
public Bits bits() throws IOException {
return null;
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestSortedSetSelector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestSortedSetSelector.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestSortedSetSelector.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestSortedSetSelector.java Mon Jun 16 19:26:19 2014
@@ -18,8 +18,7 @@ package org.apache.lucene.search;
*/
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.diskdv.DiskDocValuesFormat;
-import org.apache.lucene.codecs.lucene45.Lucene45DocValuesFormat;
+import org.apache.lucene.codecs.lucene49.Lucene49DocValuesFormat;
import org.apache.lucene.codecs.memory.DirectDocValuesFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -47,11 +46,10 @@ public class TestSortedSetSelector exten
public static void beforeClass() throws Exception {
savedCodec = Codec.getDefault();
// currently only these codecs that support random access ordinals
- int victim = random().nextInt(3);
+ int victim = random().nextInt(2);
switch(victim) {
case 0: Codec.setDefault(TestUtil.alwaysDocValuesFormat(new DirectDocValuesFormat())); break;
- case 1: Codec.setDefault(TestUtil.alwaysDocValuesFormat(new DiskDocValuesFormat())); break;
- default: Codec.setDefault(TestUtil.alwaysDocValuesFormat(new Lucene45DocValuesFormat()));
+ default: Codec.setDefault(TestUtil.alwaysDocValuesFormat(new Lucene49DocValuesFormat()));
}
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/StressRamUsageEstimator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/StressRamUsageEstimator.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/StressRamUsageEstimator.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/StressRamUsageEstimator.java Mon Jun 16 19:26:19 2014
@@ -58,7 +58,7 @@ public class StressRamUsageEstimator ext
while (true) {
// Check the current memory consumption and provide the estimate.
long jvmUsed = memoryMXBean.getHeapMemoryUsage().getUsed();
- long estimated = RamUsageEstimator.sizeOf(first);
+ long estimated = RamUsageTester.sizeOf(first);
System.out.println(String.format(Locale.ROOT, "%10d, %10d",
jvmUsed, estimated));
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestDocIdBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestDocIdBitSet.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestDocIdBitSet.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestDocIdBitSet.java Mon Jun 16 19:26:19 2014
@@ -3,6 +3,8 @@ package org.apache.lucene.util;
import java.io.IOException;
import java.util.BitSet;
+import org.junit.Ignore;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -26,5 +28,11 @@ public class TestDocIdBitSet extends Bas
public DocIdBitSet copyOf(BitSet bs, int length) throws IOException {
return new DocIdBitSet((BitSet) bs.clone());
}
-
+
+ @Override
+ @Ignore("no access to the internals of this impl")
+ public void testRamBytesUsed() throws IOException {
+ super.testRamBytesUsed();
+ }
+
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestIdentityHashSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestIdentityHashSet.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestIdentityHashSet.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestIdentityHashSet.java Mon Jun 16 19:26:19 2014
@@ -28,7 +28,7 @@ public class TestIdentityHashSet extends
Random rnd = random();
Set<Object> jdk = Collections.newSetFromMap(
new IdentityHashMap<Object,Boolean>());
- RamUsageEstimator.IdentityHashSet<Object> us = new RamUsageEstimator.IdentityHashSet<>();
+ RamUsageTester.IdentityHashSet<Object> us = new RamUsageTester.IdentityHashSet<>();
int max = 100000;
int threshold = 256;
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java Mon Jun 16 19:26:19 2014
@@ -18,11 +18,10 @@ package org.apache.lucene.util;
*/
import static org.apache.lucene.util.RamUsageEstimator.*;
+import static org.apache.lucene.util.RamUsageTester.sizeOf;
import java.util.Random;
-import com.carrotsearch.randomizedtesting.generators.RandomStrings;
-
public class TestRamUsageEstimator extends LuceneTestCase {
public void testSanity() {
assertTrue(sizeOf(new String("test string")) > shallowSizeOfInstance(String.class));
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimatorOnWildAnimals.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimatorOnWildAnimals.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimatorOnWildAnimals.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimatorOnWildAnimals.java Mon Jun 16 19:26:19 2014
@@ -40,7 +40,7 @@ public class TestRamUsageEstimatorOnWild
for (int i = 0; i < mid; i++) {
last = (last.next = new ListElement());
}
- RamUsageEstimator.sizeOf(first); // cause SOE or pass.
+ RamUsageTester.sizeOf(first); // cause SOE or pass.
lower = mid;
} catch (StackOverflowError e) {
upper = mid;
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java Mon Jun 16 19:26:19 2014
@@ -25,7 +25,6 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.packed.DirectReader;
import org.apache.lucene.util.packed.DirectWriter;
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java Mon Jun 16 19:26:19 2014
@@ -38,6 +38,7 @@ import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.RamUsageTester;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.packed.PackedInts.Reader;
import org.junit.Ignore;
@@ -69,6 +70,9 @@ public class TestPackedInts extends Luce
assertEquals(61, PackedInts.bitsRequired(0x1FFFFFFFFFFFFFFFL));
assertEquals(62, PackedInts.bitsRequired(0x3FFFFFFFFFFFFFFFL));
assertEquals(63, PackedInts.bitsRequired(0x7FFFFFFFFFFFFFFFL));
+ assertEquals(64, PackedInts.unsignedBitsRequired(-1));
+ assertEquals(64, PackedInts.unsignedBitsRequired(Long.MIN_VALUE));
+ assertEquals(1, PackedInts.bitsRequired(0));
}
public void testMaxValues() {
@@ -146,7 +150,7 @@ public class TestPackedInts extends Luce
}
in.close();
- final long expectedBytesUsed = RamUsageEstimator.sizeOf(r);
+ final long expectedBytesUsed = RamUsageTester.sizeOf(r);
final long computedBytesUsed = r.ramBytesUsed();
assertEquals(r.getClass() + "expected " + expectedBytesUsed + ", got: " + computedBytesUsed,
expectedBytesUsed, computedBytesUsed);
@@ -687,7 +691,7 @@ public class TestPackedInts extends Luce
assertEquals(10, wrt.get(7));
assertEquals(99, wrt.get(valueCount - 10));
assertEquals(1 << 10, wrt.get(valueCount - 1));
- assertEquals(RamUsageEstimator.sizeOf(wrt), wrt.ramBytesUsed());
+ assertEquals(RamUsageTester.sizeOf(wrt), wrt.ramBytesUsed());
}
public void testPagedGrowableWriter() {
@@ -716,7 +720,7 @@ public class TestPackedInts extends Luce
}
// test ramBytesUsed
- assertEquals(RamUsageEstimator.sizeOf(writer), writer.ramBytesUsed(), 8);
+ assertEquals(RamUsageTester.sizeOf(writer), writer.ramBytesUsed(), 8);
// test copy
PagedGrowableWriter copy = writer.resize(TestUtil.nextLong(random(), writer.size() / 2, writer.size() * 3 / 2));
@@ -764,7 +768,7 @@ public class TestPackedInts extends Luce
}
// test ramBytesUsed
- assertEquals(RamUsageEstimator.sizeOf(writer) - RamUsageEstimator.sizeOf(writer.format), writer.ramBytesUsed());
+ assertEquals(RamUsageTester.sizeOf(writer) - RamUsageTester.sizeOf(writer.format), writer.ramBytesUsed());
// test copy
PagedMutable copy = writer.resize(TestUtil.nextLong(random(), writer.size() / 2, writer.size() * 3 / 2));
@@ -1048,7 +1052,7 @@ public class TestPackedInts extends Luce
}
}
- final long expectedBytesUsed = RamUsageEstimator.sizeOf(buf);
+ final long expectedBytesUsed = RamUsageTester.sizeOf(buf);
final long computedBytesUsed = buf.ramBytesUsed();
assertEquals(expectedBytesUsed, computedBytesUsed);
}
Modified: lucene/dev/branches/lucene5752/lucene/expressions/src/java/org/apache/lucene/expressions/js/Javascript.g
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/expressions/src/java/org/apache/lucene/expressions/js/Javascript.g?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/expressions/src/java/org/apache/lucene/expressions/js/Javascript.g (original)
+++ lucene/dev/branches/lucene5752/lucene/expressions/src/java/org/apache/lucene/expressions/js/Javascript.g Mon Jun 16 19:26:19 2014
@@ -336,7 +336,7 @@ NAMESPACE_ID
fragment
ID
- : ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_')*
+ : ('a'..'z'|'A'..'Z'|'_'|'$') ('a'..'z'|'A'..'Z'|'0'..'9'|'_'|'$')*
;
WS
Modified: lucene/dev/branches/lucene5752/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptLexer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptLexer.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptLexer.java (original)
+++ lucene/dev/branches/lucene5752/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptLexer.java Mon Jun 16 19:26:19 2014
@@ -679,10 +679,10 @@ class JavascriptLexer extends Lexer {
// $ANTLR start "ID"
public final void mID() throws RecognitionException {
try {
- // src/java/org/apache/lucene/expressions/js/Javascript.g:340:5: ( ( 'a' .. 'z' | 'A' .. 'Z' | '_' ) ( 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '_' )* )
- // src/java/org/apache/lucene/expressions/js/Javascript.g:340:7: ( 'a' .. 'z' | 'A' .. 'Z' | '_' ) ( 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '_' )*
+ // src/java/org/apache/lucene/expressions/js/Javascript.g:340:5: ( ( 'a' .. 'z' | 'A' .. 'Z' | '_' | '$' ) ( 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '_' | '$' )* )
+ // src/java/org/apache/lucene/expressions/js/Javascript.g:340:7: ( 'a' .. 'z' | 'A' .. 'Z' | '_' | '$' ) ( 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '_' | '$' )*
{
- if ( (input.LA(1) >= 'A' && input.LA(1) <= 'Z')||input.LA(1)=='_'||(input.LA(1) >= 'a' && input.LA(1) <= 'z') ) {
+ if ( input.LA(1)=='$'||(input.LA(1) >= 'A' && input.LA(1) <= 'Z')||input.LA(1)=='_'||(input.LA(1) >= 'a' && input.LA(1) <= 'z') ) {
input.consume();
}
else {
@@ -690,12 +690,12 @@ class JavascriptLexer extends Lexer {
recover(mse);
throw mse;
}
- // src/java/org/apache/lucene/expressions/js/Javascript.g:340:31: ( 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '_' )*
+ // src/java/org/apache/lucene/expressions/js/Javascript.g:340:35: ( 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '_' | '$' )*
loop2:
while (true) {
int alt2=2;
int LA2_0 = input.LA(1);
- if ( ((LA2_0 >= '0' && LA2_0 <= '9')||(LA2_0 >= 'A' && LA2_0 <= 'Z')||LA2_0=='_'||(LA2_0 >= 'a' && LA2_0 <= 'z')) ) {
+ if ( (LA2_0=='$'||(LA2_0 >= '0' && LA2_0 <= '9')||(LA2_0 >= 'A' && LA2_0 <= 'Z')||LA2_0=='_'||(LA2_0 >= 'a' && LA2_0 <= 'z')) ) {
alt2=1;
}
@@ -703,7 +703,7 @@ class JavascriptLexer extends Lexer {
case 1 :
// src/java/org/apache/lucene/expressions/js/Javascript.g:
{
- if ( (input.LA(1) >= '0' && input.LA(1) <= '9')||(input.LA(1) >= 'A' && input.LA(1) <= 'Z')||input.LA(1)=='_'||(input.LA(1) >= 'a' && input.LA(1) <= 'z') ) {
+ if ( input.LA(1)=='$'||(input.LA(1) >= '0' && input.LA(1) <= '9')||(input.LA(1) >= 'A' && input.LA(1) <= 'Z')||input.LA(1)=='_'||(input.LA(1) >= 'a' && input.LA(1) <= 'z') ) {
input.consume();
}
else {
@@ -1476,6 +1476,7 @@ class JavascriptLexer extends Lexer {
alt17=27;
}
break;
+ case '$':
case 'A':
case 'B':
case 'C':
Modified: lucene/dev/branches/lucene5752/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptParser.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptParser.java (original)
+++ lucene/dev/branches/lucene5752/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptParser.java Mon Jun 16 19:26:19 2014
@@ -1571,7 +1571,7 @@ class JavascriptParser extends Parser {
stream_arguments.add(arguments48.getTree());
// AST REWRITE
- // elements: NAMESPACE_ID, arguments
+ // elements: arguments, NAMESPACE_ID
// token labels:
// rule labels: retval
// token list labels:
Modified: lucene/dev/branches/lucene5752/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java (original)
+++ lucene/dev/branches/lucene5752/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java Mon Jun 16 19:26:19 2014
@@ -136,6 +136,25 @@ public class TestDemoExpressions extend
}
}
+ /** Uses variables with $ */
+ public void testDollarVariable() throws Exception {
+ Expression expr = JavascriptCompiler.compile("$0+$score");
+
+ SimpleBindings bindings = new SimpleBindings();
+ bindings.add(new SortField("$0", SortField.Type.SCORE));
+ bindings.add(new SortField("$score", SortField.Type.SCORE));
+
+ Sort sort = new Sort(expr.getSortField(bindings, true));
+ Query query = new TermQuery(new Term("body", "contents"));
+ TopFieldDocs td = searcher.search(query, null, 3, sort, true, true);
+ for (int i = 0; i < 3; i++) {
+ FieldDoc d = (FieldDoc) td.scoreDocs[i];
+ float expected = 2*d.score;
+ float actual = ((Double)d.fields[0]).floatValue();
+ assertEquals(expected, actual, CheckHits.explainToleranceDelta(expected, actual));
+ }
+ }
+
/** tests expression referring to another expression */
public void testExpressionRefersToExpression() throws Exception {
Expression expr1 = JavascriptCompiler.compile("_score");
Modified: lucene/dev/branches/lucene5752/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java (original)
+++ lucene/dev/branches/lucene5752/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java Mon Jun 16 19:26:19 2014
@@ -163,6 +163,11 @@ public final class DoubleRange extends R
public DocIdSetIterator iterator() {
throw new UnsupportedOperationException("this filter can only be accessed via bits()");
}
+
+ @Override
+ public long ramBytesUsed() {
+ return 0L;
+ }
};
}
};
Modified: lucene/dev/branches/lucene5752/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java (original)
+++ lucene/dev/branches/lucene5752/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java Mon Jun 16 19:26:19 2014
@@ -155,6 +155,11 @@ public final class LongRange extends Ran
public DocIdSetIterator iterator() {
throw new UnsupportedOperationException("this filter can only be accessed via bits()");
}
+
+ @Override
+ public long ramBytesUsed() {
+ return 0L;
+ }
};
}
};
Modified: lucene/dev/branches/lucene5752/lucene/join/src/java/org/apache/lucene/search/join/FixedBitSetCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/join/src/java/org/apache/lucene/search/join/FixedBitSetCachingWrapperFilter.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/join/src/java/org/apache/lucene/search/join/FixedBitSetCachingWrapperFilter.java (original)
+++ lucene/dev/branches/lucene5752/lucene/join/src/java/org/apache/lucene/search/join/FixedBitSetCachingWrapperFilter.java Mon Jun 16 19:26:19 2014
@@ -17,6 +17,8 @@ package org.apache.lucene.search.join;
* limitations under the License.
*/
+import static org.apache.lucene.search.DocIdSet.EMPTY;
+
import java.io.IOException;
import org.apache.lucene.index.AtomicReader;
@@ -39,7 +41,7 @@ public final class FixedBitSetCachingWra
protected DocIdSet docIdSetToCache(DocIdSet docIdSet, AtomicReader reader)
throws IOException {
if (docIdSet == null) {
- return EMPTY_DOCIDSET;
+ return EMPTY;
} else if (docIdSet instanceof FixedBitSet) {
// this is different from CachingWrapperFilter: even when the DocIdSet is
// cacheable, we convert it to a FixedBitSet since we require all the
@@ -48,7 +50,7 @@ public final class FixedBitSetCachingWra
} else {
final DocIdSetIterator it = docIdSet.iterator();
if (it == null) {
- return EMPTY_DOCIDSET;
+ return EMPTY;
} else {
final FixedBitSet copy = new FixedBitSet(reader.maxDoc());
copy.or(it);
Modified: lucene/dev/branches/lucene5752/lucene/memory/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/memory/build.xml?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/memory/build.xml (original)
+++ lucene/dev/branches/lucene5752/lucene/memory/build.xml Mon Jun 16 19:26:19 2014
@@ -19,10 +19,6 @@
<project name="memory" default="default">
- <property name="forbidden-rue-excludes" value="
- org/apache/lucene/index/memory/MemoryIndex.class
- "/>
-
<description>
Single-document in-memory index implementation
</description>
Modified: lucene/dev/branches/lucene5752/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/lucene5752/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Mon Jun 16 19:26:19 2014
@@ -33,7 +33,6 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
@@ -45,12 +44,12 @@ import org.apache.lucene.index.Fields;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.OrdTermState;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.Collector;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
@@ -573,15 +572,6 @@ public class MemoryIndex {
*/
}
}
-
- /**
- * Returns a reasonable approximation of the main memory [bytes] consumed by
- * this instance. Useful for smart memory sensititive caches/pools.
- * @return the main memory consumption
- */
- public long getMemorySize() {
- return RamUsageEstimator.sizeOf(this);
- }
/** sorts into ascending order (on demand), reusing memory along the way */
private void sortFields() {
@@ -654,7 +644,6 @@ public class MemoryIndex {
result.append("\tterms=" + info.terms.size());
result.append(", positions=" + numPositions);
- result.append(", memory=" + RamUsageEstimator.humanReadableUnits(RamUsageEstimator.sizeOf(info)));
result.append("\n");
sumPositions += numPositions;
sumTerms += info.terms.size();
@@ -663,7 +652,6 @@ public class MemoryIndex {
result.append("\nfields=" + sortedFields.length);
result.append(", terms=" + sumTerms);
result.append(", positions=" + sumPositions);
- result.append(", memory=" + RamUsageEstimator.humanReadableUnits(getMemorySize()));
return result.toString();
}
@@ -794,6 +782,11 @@ public class MemoryIndex {
}
@Override
+ public SortedNumericDocValues getSortedNumericDocValues(String field) {
+ return null;
+ }
+
+ @Override
public SortedSetDocValues getSortedSetDocValues(String field) {
return null;
}
Modified: lucene/dev/branches/lucene5752/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (original)
+++ lucene/dev/branches/lucene5752/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java Mon Jun 16 19:26:19 2014
@@ -161,14 +161,6 @@ public class MemoryIndexTest extends Bas
memory.addField("foo", fooField.toString(), analyzer);
memory.addField("term", termField.toString(), analyzer);
- if (VERBOSE) {
- System.out.println("Random MemoryIndex:\n" + memory.toString());
- System.out.println("Same index as RAMDirectory: " +
- RamUsageEstimator.humanReadableUnits(RamUsageEstimator.sizeOf(ramdir)));
- System.out.println();
- } else {
- assertTrue(memory.getMemorySize() > 0L);
- }
AtomicReader reader = (AtomicReader) memory.createSearcher().getIndexReader();
DirectoryReader competitor = DirectoryReader.open(ramdir);
duellReaders(competitor, reader);
Modified: lucene/dev/branches/lucene5752/lucene/misc/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/misc/build.xml?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/misc/build.xml (original)
+++ lucene/dev/branches/lucene5752/lucene/misc/build.xml Mon Jun 16 19:26:19 2014
@@ -32,10 +32,6 @@
org/apache/lucene/misc/IndexMergeTool.class
"/>
- <property name="forbidden-rue-excludes" value="
- org/apache/lucene/uninverting/FieldCache$CacheEntry.class
- "/>
-
<import file="../module-build.xml"/>
<target name="install-cpptasks" unless="cpptasks.uptodate" depends="ivy-availability-check,ivy-fail,ivy-configure">
Modified: lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java (original)
+++ lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java Mon Jun 16 19:26:19 2014
@@ -33,6 +33,7 @@ import org.apache.lucene.index.SortedSet
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.PagedBytes;
@@ -110,7 +111,7 @@ import org.apache.lucene.util.StringHelp
*
*/
-public class DocTermOrds {
+public class DocTermOrds implements Accountable {
// Term ords are shifted by this, internally, to reserve
// values 0 (end term) and 1 (index is a pointer into byte array)
@@ -167,7 +168,7 @@ public class DocTermOrds {
protected DocsEnum docsEnum;
/** Returns total bytes used. */
- public long ramUsedInBytes() {
+ public long ramBytesUsed() {
// can cache the mem size since it shouldn't change
if (memsz!=0) return memsz;
long sz = 8*8 + 32; // local fields
Modified: lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java (original)
+++ lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java Mon Jun 16 19:26:19 2014
@@ -34,6 +34,7 @@ import org.apache.lucene.index.SortedDoc
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
@@ -54,8 +55,14 @@ interface FieldCache {
/**
* Placeholder indicating creation of this cache is currently in-progress.
*/
- public static final class CreationPlaceholder {
- Object value;
+ public static final class CreationPlaceholder implements Accountable {
+ Accountable value;
+
+ @Override
+ public long ramBytesUsed() {
+ // don't call on the in-progress value, might make things angry.
+ return RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+ }
}
/**
@@ -270,13 +277,12 @@ interface FieldCache {
private final String fieldName;
private final Class<?> cacheType;
private final Object custom;
- private final Object value;
- private String size;
+ private final Accountable value;
public CacheEntry(Object readerKey, String fieldName,
Class<?> cacheType,
Object custom,
- Object value) {
+ Accountable value) {
this.readerKey = readerKey;
this.fieldName = fieldName;
this.cacheType = cacheType;
@@ -304,21 +310,13 @@ interface FieldCache {
return value;
}
- /**
- * Computes (and stores) the estimated size of the cache Value
- * @see #getEstimatedSize
- */
- public void estimateSize() {
- long bytesUsed = RamUsageEstimator.sizeOf(getValue());
- size = RamUsageEstimator.humanReadableUnits(bytesUsed);
- }
-
/**
* The most recently estimated size of the value, null unless
* estimateSize has been called.
*/
public String getEstimatedSize() {
- return size;
+ long bytesUsed = value == null ? 0L : value.ramBytesUsed();
+ return RamUsageEstimator.humanReadableUnits(bytesUsed);
}
@Override
@@ -331,9 +329,7 @@ interface FieldCache {
b.append(System.identityHashCode(getValue()));
String s = getEstimatedSize();
- if(null != s) {
- b.append(" (size =~ ").append(s).append(')');
- }
+ b.append(" (size =~ ").append(s).append(')');
return b.toString();
}
Modified: lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java (original)
+++ lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java Mon Jun 16 19:26:19 2014
@@ -38,10 +38,12 @@ import org.apache.lucene.index.SortedSet
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.PagedBytes;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.GrowableWriter;
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
import org.apache.lucene.util.packed.PackedInts;
@@ -87,11 +89,11 @@ class FieldCacheImpl implements FieldCac
final Cache cache = cacheEntry.getValue();
final Class<?> cacheType = cacheEntry.getKey();
synchronized(cache.readerCache) {
- for (final Map.Entry<Object,Map<CacheKey, Object>> readerCacheEntry : cache.readerCache.entrySet()) {
+ for (final Map.Entry<Object,Map<CacheKey, Accountable>> readerCacheEntry : cache.readerCache.entrySet()) {
final Object readerKey = readerCacheEntry.getKey();
if (readerKey == null) continue;
- final Map<CacheKey, Object> innerCache = readerCacheEntry.getValue();
- for (final Map.Entry<CacheKey, Object> mapEntry : innerCache.entrySet()) {
+ final Map<CacheKey, Accountable> innerCache = readerCacheEntry.getValue();
+ for (final Map.Entry<CacheKey, Accountable> mapEntry : innerCache.entrySet()) {
CacheKey entry = mapEntry.getKey();
result.add(new CacheEntry(readerKey, entry.field,
cacheType, entry.custom,
@@ -124,9 +126,9 @@ class FieldCacheImpl implements FieldCac
final FieldCacheImpl wrapper;
- final Map<Object,Map<CacheKey,Object>> readerCache = new WeakHashMap<>();
+ final Map<Object,Map<CacheKey,Accountable>> readerCache = new WeakHashMap<>();
- protected abstract Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField)
+ protected abstract Accountable createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField)
throws IOException;
/** Remove this reader from the cache, if present. */
@@ -138,10 +140,10 @@ class FieldCacheImpl implements FieldCac
/** Sets the key to the value for the provided reader;
* if the key is already set then this doesn't change it. */
- public void put(AtomicReader reader, CacheKey key, Object value) {
+ public void put(AtomicReader reader, CacheKey key, Accountable value) {
final Object readerKey = reader.getCoreCacheKey();
synchronized (readerCache) {
- Map<CacheKey,Object> innerCache = readerCache.get(readerKey);
+ Map<CacheKey,Accountable> innerCache = readerCache.get(readerKey);
if (innerCache == null) {
// First time this reader is using FieldCache
innerCache = new HashMap<>();
@@ -158,8 +160,8 @@ class FieldCacheImpl implements FieldCac
}
public Object get(AtomicReader reader, CacheKey key, boolean setDocsWithField) throws IOException {
- Map<CacheKey,Object> innerCache;
- Object value;
+ Map<CacheKey,Accountable> innerCache;
+ Accountable value;
final Object readerKey = reader.getCoreCacheKey();
synchronized (readerCache) {
innerCache = readerCache.get(readerKey);
@@ -324,7 +326,7 @@ class FieldCacheImpl implements FieldCac
} else {
bits = docsWithField;
}
- caches.get(DocsWithFieldCache.class).put(reader, new CacheKey(field, null), bits);
+ caches.get(DocsWithFieldCache.class).put(reader, new CacheKey(field, null), new BitsEntry(bits));
}
private static class HoldsOneThing<T> {
@@ -358,7 +360,26 @@ class FieldCacheImpl implements FieldCac
} else if (!fieldInfo.isIndexed()) {
return new Bits.MatchNoBits(reader.maxDoc());
}
- return (Bits) caches.get(DocsWithFieldCache.class).get(reader, new CacheKey(field, null), false);
+ BitsEntry bitsEntry = (BitsEntry) caches.get(DocsWithFieldCache.class).get(reader, new CacheKey(field, null), false);
+ return bitsEntry.bits;
+ }
+
+ static class BitsEntry implements Accountable {
+ final Bits bits;
+
+ BitsEntry(Bits bits) {
+ this.bits = bits;
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ long base = RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+ if (bits instanceof Bits.MatchAllBits || bits instanceof Bits.MatchNoBits) {
+ return base;
+ } else {
+ return base + (bits.length() >>> 3);
+ }
+ }
}
static final class DocsWithFieldCache extends Cache {
@@ -367,7 +388,7 @@ class FieldCacheImpl implements FieldCac
}
@Override
- protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
+ protected BitsEntry createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
throws IOException {
final String field = key.field;
final int maxDoc = reader.maxDoc();
@@ -380,7 +401,7 @@ class FieldCacheImpl implements FieldCac
assert termsDocCount <= maxDoc;
if (termsDocCount == maxDoc) {
// Fast case: all docs have this field:
- return new Bits.MatchAllBits(maxDoc);
+ return new BitsEntry(new Bits.MatchAllBits(maxDoc));
}
final TermsEnum termsEnum = terms.iterator(null);
DocsEnum docs = null;
@@ -406,15 +427,15 @@ class FieldCacheImpl implements FieldCac
}
}
if (res == null) {
- return new Bits.MatchNoBits(maxDoc);
+ return new BitsEntry(new Bits.MatchNoBits(maxDoc));
}
final int numSet = res.cardinality();
if (numSet >= maxDoc) {
// The cardinality of the BitSet is maxDoc if all documents have a value.
assert numSet == maxDoc;
- return new Bits.MatchAllBits(maxDoc);
+ return new BitsEntry(new Bits.MatchAllBits(maxDoc));
}
- return res;
+ return new BitsEntry(res);
}
}
@@ -441,7 +462,7 @@ class FieldCacheImpl implements FieldCac
}
}
- static class LongsFromArray extends NumericDocValues {
+ static class LongsFromArray extends NumericDocValues implements Accountable {
private final PackedInts.Reader values;
private final long minValue;
@@ -454,6 +475,11 @@ class FieldCacheImpl implements FieldCac
public long get(int docID) {
return minValue + values.get(docID);
}
+
+ @Override
+ public long ramBytesUsed() {
+ return values.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_LONG;
+ }
}
static final class LongCache extends Cache {
@@ -462,7 +488,7 @@ class FieldCacheImpl implements FieldCac
}
@Override
- protected Object createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
+ protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
throws IOException {
final Parser parser = (Parser) key.custom;
@@ -523,7 +549,7 @@ class FieldCacheImpl implements FieldCac
}
}
- public static class SortedDocValuesImpl {
+ public static class SortedDocValuesImpl implements Accountable {
private final PagedBytes.Reader bytes;
private final MonotonicAppendingLongBuffer termOrdToBytesOffset;
private final PackedInts.Reader docToTermOrd;
@@ -563,6 +589,15 @@ class FieldCacheImpl implements FieldCac
}
};
}
+
+ @Override
+ public long ramBytesUsed() {
+ return bytes.ramBytesUsed() +
+ termOrdToBytesOffset.ramBytesUsed() +
+ docToTermOrd.ramBytesUsed() +
+ 3*RamUsageEstimator.NUM_BYTES_OBJECT_REF +
+ RamUsageEstimator.NUM_BYTES_INT;
+ }
}
public SortedDocValues getTermsIndex(AtomicReader reader, String field) throws IOException {
@@ -597,7 +632,7 @@ class FieldCacheImpl implements FieldCac
}
@Override
- protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
+ protected Accountable createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
throws IOException {
final int maxDoc = reader.maxDoc();
@@ -679,7 +714,7 @@ class FieldCacheImpl implements FieldCac
}
}
- private static class BinaryDocValuesImpl {
+ private static class BinaryDocValuesImpl implements Accountable {
private final PagedBytes.Reader bytes;
private final PackedInts.Reader docToOffset;
@@ -703,6 +738,11 @@ class FieldCacheImpl implements FieldCac
}
};
}
+
+ @Override
+ public long ramBytesUsed() {
+ return bytes.ramBytesUsed() + docToOffset.ramBytesUsed() + 2*RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+ }
}
// TODO: this if DocTermsIndex was already created, we
@@ -742,7 +782,7 @@ class FieldCacheImpl implements FieldCac
}
@Override
- protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField)
+ protected Accountable createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField)
throws IOException {
// TODO: would be nice to first check if DocTermsIndex
@@ -880,7 +920,7 @@ class FieldCacheImpl implements FieldCac
}
@Override
- protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
+ protected Accountable createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
throws IOException {
BytesRef prefix = (BytesRef) key.custom;
return new DocTermOrds(reader, null, key.field, prefix);
Modified: lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheSanityChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheSanityChecker.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheSanityChecker.java (original)
+++ lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheSanityChecker.java Mon Jun 16 19:26:19 2014
@@ -27,7 +27,6 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.uninverting.FieldCache.CacheEntry;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.MapOfSets;
/**
@@ -54,20 +53,10 @@ import org.apache.lucene.util.MapOfSets;
*/
final class FieldCacheSanityChecker {
- private boolean estimateRam;
-
public FieldCacheSanityChecker() {
/* NOOP */
}
- /**
- * If set, estimate size for all CacheEntry objects will be calculateed.
- */
- public void setRamUsageEstimator(boolean flag) {
- estimateRam = flag;
- }
-
-
/**
* Quick and dirty convenience method
* @see #check
@@ -83,7 +72,6 @@ final class FieldCacheSanityChecker {
*/
public static Insanity[] checkSanity(CacheEntry... cacheEntries) {
FieldCacheSanityChecker sanityChecker = new FieldCacheSanityChecker();
- sanityChecker.setRamUsageEstimator(true);
return sanityChecker.check(cacheEntries);
}
@@ -99,12 +87,6 @@ final class FieldCacheSanityChecker {
if (null == cacheEntries || 0 == cacheEntries.length)
return new Insanity[0];
- if (estimateRam) {
- for (int i = 0; i < cacheEntries.length; i++) {
- cacheEntries[i].estimateSize();
- }
- }
-
// the indirect mapping lets MapOfSet dedup identical valIds for us
//
// maps the (valId) identityhashCode of cache values to
@@ -125,7 +107,7 @@ final class FieldCacheSanityChecker {
// It's OK to have dup entries, where one is eg
// float[] and the other is the Bits (from
// getDocWithField())
- if (val instanceof Bits) {
+ if (val instanceof FieldCacheImpl.BitsEntry) {
continue;
}
Modified: lucene/dev/branches/lucene5752/lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java Mon Jun 16 19:26:19 2014
@@ -235,6 +235,10 @@ public class TestUninvertingReader exten
break;
case SORTED_SET:
reader.getSortedSetDocValues(field);
+ break;
+ case SORTED_NUMERIC:
+ // not supported
+ return;
default:
throw new AssertionError();
}
Modified: lucene/dev/branches/lucene5752/lucene/queries/src/java/org/apache/lucene/queries/TermFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/queries/src/java/org/apache/lucene/queries/TermFilter.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/queries/src/java/org/apache/lucene/queries/TermFilter.java (original)
+++ lucene/dev/branches/lucene5752/lucene/queries/src/java/org/apache/lucene/queries/TermFilter.java Mon Jun 16 19:26:19 2014
@@ -72,6 +72,10 @@ final public class TermFilter extends Fi
return termsEnum.docs(acceptDocs, null, DocsEnum.FLAG_NONE);
}
+ @Override
+ public long ramBytesUsed() {
+ return 0L;
+ }
};
}
Modified: lucene/dev/branches/lucene5752/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java (original)
+++ lucene/dev/branches/lucene5752/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java Mon Jun 16 19:26:19 2014
@@ -111,17 +111,7 @@ public class BooleanFilterTest extends L
return new Filter() {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) {
- return new DocIdSet() {
- @Override
- public DocIdSetIterator iterator() {
- return null;
- }
-
- @Override
- public boolean isCacheable() {
- return true;
- }
- };
+ return DocIdSet.EMPTY;
}
};
}
Modified: lucene/dev/branches/lucene5752/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java (original)
+++ lucene/dev/branches/lucene5752/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java Mon Jun 16 19:26:19 2014
@@ -146,7 +146,7 @@ public class TestDocValuesFieldSources e
public void test() throws IOException {
for (DocValuesType type : DocValuesType.values()) {
- if (type != DocValuesType.SORTED_SET) {
+ if (type != DocValuesType.SORTED_SET && type != DocValuesType.SORTED_NUMERIC) {
test(type);
}
}
Modified: lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/ContainsPrefixTreeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/ContainsPrefixTreeFilter.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/ContainsPrefixTreeFilter.java (original)
+++ lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/ContainsPrefixTreeFilter.java Mon Jun 16 19:26:19 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.spatial.prefix
import com.spatial4j.core.shape.Shape;
import com.spatial4j.core.shape.SpatialRelation;
+
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.search.DocIdSet;
@@ -28,6 +29,7 @@ import org.apache.lucene.spatial.prefix.
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.SentinelIntSet;
import java.io.IOException;
@@ -299,6 +301,14 @@ public class ContainsPrefixTreeFilter ex
};
}
+ @Override
+ public long ramBytesUsed() {
+ return RamUsageEstimator.alignObjectSize(
+ RamUsageEstimator.NUM_BYTES_OBJECT_REF
+ + RamUsageEstimator.NUM_BYTES_INT)
+ + intSet.ramBytesUsed();
+ }
+
}//class SmallDocSet
}
Modified: lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/NumberRangePrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/NumberRangePrefixTree.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/NumberRangePrefixTree.java (original)
+++ lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/NumberRangePrefixTree.java Mon Jun 16 19:26:19 2014
@@ -96,17 +96,29 @@ public abstract class NumberRangePrefixT
protected class NRShape implements Shape {
private final LevelledValue minLV, maxLV;
+ private final int lastLevelInCommon;//computed; not part of identity
/** Don't call directly; see {@link #toRangeShape(com.spatial4j.core.shape.Shape, com.spatial4j.core.shape.Shape)}. */
private NRShape(LevelledValue minLV, LevelledValue maxLV) {
this.minLV = minLV;
this.maxLV = maxLV;
+
+ //calc lastLevelInCommon
+ int level = 1;
+ for (; level <= minLV.getLevel() && level <= maxLV.getLevel(); level++) {
+ if (minLV.getValAtLevel(level) != maxLV.getValAtLevel(level))
+ break;
+ }
+ lastLevelInCommon = level - 1;
}
public LevelledValue getMinLV() { return minLV; }
public LevelledValue getMaxLV() { return maxLV; }
+ /** How many levels are in common between minLV and maxLV. */
+ private int getLastLevelInCommon() { return lastLevelInCommon; }
+
@Override
public SpatialRelation relate(Shape shape) {
// if (shape instanceof LevelledValue)
@@ -561,11 +573,11 @@ public abstract class NumberRangePrefixT
private void initIter(Shape filter) {
cellNumber = -1;
- if (filter instanceof LevelledValue && ((LevelledValue)filter).getLevel() == 0)
+ if (filter instanceof LevelledValue && ((LevelledValue) filter).getLevel() == 0)
filter = null;//world means everything -- no filter
iterFilter = filter;
- NRCell parent = getLVAtLevel(getLevel()-1);
+ NRCell parent = getLVAtLevel(getLevel() - 1);
// Initialize iter* members.
@@ -580,17 +592,22 @@ public abstract class NumberRangePrefixT
final LevelledValue minLV;
final LevelledValue maxLV;
+ final int lastLevelInCommon;//between minLV & maxLV
if (filter instanceof NRShape) {
NRShape nrShape = (NRShape) iterFilter;
minLV = nrShape.getMinLV();
maxLV = nrShape.getMaxLV();
+ lastLevelInCommon = nrShape.getLastLevelInCommon();
} else {
- minLV = (LevelledValue)iterFilter;
+ minLV = (LevelledValue) iterFilter;
maxLV = minLV;
+ lastLevelInCommon = minLV.getLevel();
}
- //fast path check when using same filter
- if (iterFilter == parent.iterFilter) {
+ //fast path optimization that is usually true, but never first level
+ if (iterFilter == parent.iterFilter &&
+ (getLevel() <= lastLevelInCommon || parent.iterFirstCellNumber != parent.iterLastCellNumber)) {
+ //TODO benchmark if this optimization pays off. We avoid two comparePrefixLV calls.
if (parent.iterFirstIsIntersects && parent.cellNumber == parent.iterFirstCellNumber
&& minLV.getLevel() >= getLevel()) {
iterFirstCellNumber = minLV.getValAtLevel(getLevel());
@@ -616,7 +633,7 @@ public abstract class NumberRangePrefixT
return;
}
- //uncommon to get here, except for level 1 which always happens
+ //not common to get here, except for level 1 which always happens
int startCmp = comparePrefixLV(minLV, parent);
if (startCmp > 0) {//start comes after this cell
@@ -650,6 +667,12 @@ public abstract class NumberRangePrefixT
iterLastCellNumber = maxLV.getValAtLevel(getLevel());
iterLastIsIntersects = (maxLV.getLevel() > getLevel());
}
+ if (iterFirstCellNumber == iterLastCellNumber) {
+ if (iterLastIsIntersects)
+ iterFirstIsIntersects = true;
+ else if (iterFirstIsIntersects)
+ iterLastIsIntersects = true;
+ }
}
@Override
Modified: lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java (original)
+++ lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java Mon Jun 16 19:26:19 2014
@@ -21,6 +21,7 @@ import com.spatial4j.core.context.Spatia
import com.spatial4j.core.io.BinaryCodec;
import com.spatial4j.core.shape.Point;
import com.spatial4j.core.shape.Shape;
+
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.AtomicReaderContext;
@@ -170,6 +171,11 @@ public class SerializedDVStrategy extend
}
};
}
+
+ @Override
+ public long ramBytesUsed() {
+ return 0L;
+ }
};
}
Modified: lucene/dev/branches/lucene5752/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java (original)
+++ lucene/dev/branches/lucene5752/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java Mon Jun 16 19:26:19 2014
@@ -21,7 +21,7 @@ import com.spatial4j.core.context.Spatia
import com.spatial4j.core.shape.Point;
import com.spatial4j.core.shape.Rectangle;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.codecs.lucene45.Lucene45DocValuesFormat;
+import org.apache.lucene.codecs.lucene49.Lucene49DocValuesFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriterConfig;
@@ -90,7 +90,7 @@ public abstract class SpatialTestCase ex
final IndexWriterConfig indexWriterConfig = LuceneTestCase.newIndexWriterConfig(random, LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(random));
//TODO can we randomly choose a doc-values supported format?
if (needsDocValues())
- indexWriterConfig.setCodec( TestUtil.alwaysDocValuesFormat(new Lucene45DocValuesFormat()));;
+ indexWriterConfig.setCodec( TestUtil.alwaysDocValuesFormat(new Lucene49DocValuesFormat()));;
return indexWriterConfig;
}
Modified: lucene/dev/branches/lucene5752/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/DateNRStrategyTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/DateNRStrategyTest.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/DateNRStrategyTest.java (original)
+++ lucene/dev/branches/lucene5752/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/DateNRStrategyTest.java Mon Jun 16 19:26:19 2014
@@ -91,6 +91,15 @@ public class DateNRStrategyTest extends
tree.toShape(randomCalendar()), true);
}
+ @Test
+ public void testBugInitIterOptimization() throws Exception {
+ //bug due to fast path initIter() optimization
+ testOperation(
+ tree.parseShape("[2014-03-27T23 TO 2014-04-01T01]"),
+ SpatialOperation.Intersects,
+ tree.parseShape("[2014-04 TO 2014-04-01T02]"), true);
+ }
+
@Override
protected Shape randomIndexedShape() {
Calendar cal1 = randomCalendar();
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java Mon Jun 16 19:26:19 2014
@@ -32,6 +32,7 @@ import org.apache.lucene.index.NumericDo
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -131,6 +132,30 @@ public class AssertingDocValuesFormat ex
}
@Override
+ public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ long valueCount = 0;
+ Iterator<Number> valueIterator = values.iterator();
+ for (Number count : docToValueCount) {
+ assert count != null;
+ assert count.intValue() >= 0;
+ valueCount += count.intValue();
+ long previous = Long.MIN_VALUE;
+ for (int i = 0; i < count.intValue(); i++) {
+ assert valueIterator.hasNext();
+ Number next = valueIterator.next();
+ assert next != null;
+ long nextValue = next.longValue();
+ assert nextValue >= previous;
+ previous = nextValue;
+ }
+ }
+ assert valueIterator.hasNext() == false;
+ checkIterator(docToValueCount.iterator(), maxDoc, false);
+ checkIterator(values.iterator(), valueCount, false);
+ in.addSortedNumericField(field, docToValueCount, values);
+ }
+
+ @Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
long valueCount = 0;
BytesRef lastValue = null;
@@ -217,6 +242,11 @@ public class AssertingDocValuesFormat ex
public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
throw new IllegalStateException();
}
+
+ @Override
+ public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ throw new IllegalStateException();
+ }
@Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
@@ -281,6 +311,14 @@ public class AssertingDocValuesFormat ex
}
@Override
+ public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
+ assert field.getDocValuesType() == FieldInfo.DocValuesType.SORTED_NUMERIC;
+ SortedNumericDocValues values = in.getSortedNumeric(field);
+ assert values != null;
+ return new AssertingAtomicReader.AssertingSortedNumericDocValues(values, maxDoc);
+ }
+
+ @Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
assert field.getDocValuesType() == FieldInfo.DocValuesType.SORTED_SET;
SortedSetDocValues values = in.getSortedSet(field);
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java Mon Jun 16 19:26:19 2014
@@ -24,15 +24,15 @@ import org.apache.lucene.codecs.DocValue
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat.AssertingNormsConsumer;
import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat.AssertingDocValuesProducer;
-import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
+import org.apache.lucene.codecs.lucene49.Lucene49NormsFormat;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
/**
- * Just like {@link Lucene42NormsFormat} but with additional asserts.
+ * Just like {@link Lucene49NormsFormat} but with additional asserts.
*/
public class AssertingNormsFormat extends NormsFormat {
- private final NormsFormat in = new Lucene42NormsFormat();
+ private final NormsFormat in = new Lucene49NormsFormat();
@Override
public DocValuesConsumer normsConsumer(SegmentWriteState state) throws IOException {
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java Mon Jun 16 19:26:19 2014
@@ -23,12 +23,12 @@ import org.apache.lucene.codecs.NormsFor
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.diskdv.DiskDocValuesFormat;
-import org.apache.lucene.codecs.diskdv.DiskNormsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat;
import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
import org.apache.lucene.codecs.lucene49.Lucene49Codec;
+import org.apache.lucene.codecs.lucene49.Lucene49DocValuesFormat;
+import org.apache.lucene.codecs.lucene49.Lucene49NormsFormat;
/** Codec that tries to use as little ram as possible because he spent all his money on beer */
// TODO: better name :)
@@ -40,9 +40,8 @@ public class CheapBastardCodec extends F
// uncompressing versions, waste lots of disk but no ram
private final StoredFieldsFormat storedFields = new Lucene40StoredFieldsFormat();
private final TermVectorsFormat termVectors = new Lucene40TermVectorsFormat();
- // these go to disk for all docvalues/norms datastructures
- private final DocValuesFormat docValues = new DiskDocValuesFormat();
- private final NormsFormat norms = new DiskNormsFormat();
+ private final DocValuesFormat docValues = new Lucene49DocValuesFormat();
+ private final NormsFormat norms = new Lucene49NormsFormat();
public CheapBastardCodec() {
super("CheapBastard", new Lucene49Codec());
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java Mon Jun 16 19:26:19 2014
@@ -1,9 +1,5 @@
package org.apache.lucene.codecs.compressing;
-import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
-import org.apache.lucene.util.packed.PackedInts;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -35,9 +31,4 @@ public class FastCompressingCodec extend
public FastCompressingCodec() {
this(1 << 14, false);
}
-
- @Override
- public NormsFormat normsFormat() {
- return new Lucene42NormsFormat(PackedInts.FAST);
- }
}
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java Mon Jun 16 19:26:19 2014
@@ -1,9 +1,5 @@
package org.apache.lucene.codecs.compressing;
-import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
-import org.apache.lucene.util.packed.PackedInts;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -35,9 +31,4 @@ public class FastDecompressionCompressin
public FastDecompressionCompressingCodec() {
this(1 << 14, false);
}
-
- @Override
- public NormsFormat normsFormat() {
- return new Lucene42NormsFormat(PackedInts.DEFAULT);
- }
}
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java Mon Jun 16 19:26:19 2014
@@ -1,9 +1,5 @@
package org.apache.lucene.codecs.compressing;
-import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
-import org.apache.lucene.util.packed.PackedInts;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -35,9 +31,4 @@ public class HighCompressionCompressingC
public HighCompressionCompressingCodec() {
this(1 << 14, false);
}
-
- @Override
- public NormsFormat normsFormat() {
- return new Lucene42NormsFormat(PackedInts.COMPACT);
- }
}
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyDocValuesFormat.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyDocValuesFormat.java Mon Jun 16 19:26:19 2014
@@ -93,6 +93,14 @@ class CrankyDocValuesFormat extends DocV
}
delegate.addSortedField(field, values, docToOrd);
}
+
+ @Override
+ public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException from DocValuesConsumer.addSortedNumericField()");
+ }
+ delegate.addSortedNumericField(field, docToValueCount, values);
+ }
@Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java Mon Jun 16 19:26:19 2014
@@ -531,6 +531,11 @@ class Lucene40DocValuesWriter extends Do
}
@Override
+ public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ throw new UnsupportedOperationException("Lucene 4.0 does not support SortedNumeric docvalues");
+ }
+
+ @Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
throw new UnsupportedOperationException("Lucene 4.0 does not support SortedSet docvalues");
}
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java Mon Jun 16 19:26:19 2014
@@ -378,4 +378,9 @@ class Lucene42DocValuesConsumer extends
throw new UnsupportedOperationException();
}
}
+
+ @Override
+ public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ throw new UnsupportedOperationException("Lucene 4.2 does not support SORTED_NUMERIC");
+ }
}
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42RWCodec.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42RWCodec.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42RWCodec.java Mon Jun 16 19:26:19 2014
@@ -32,7 +32,7 @@ import org.apache.lucene.util.LuceneTest
public class Lucene42RWCodec extends Lucene42Codec {
private static final DocValuesFormat dv = new Lucene42RWDocValuesFormat();
- private static final NormsFormat norms = new Lucene42NormsFormat();
+ private static final NormsFormat norms = new Lucene42RWNormsFormat();
private final FieldInfosFormat fieldInfosFormat = new Lucene42FieldInfosFormat() {
@Override
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java Mon Jun 16 19:26:19 2014
@@ -25,7 +25,7 @@ import org.apache.lucene.codecs.FieldInf
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.lucene42.Lucene42FieldInfosFormat;
import org.apache.lucene.codecs.lucene42.Lucene42FieldInfosWriter;
-import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
+import org.apache.lucene.codecs.lucene42.Lucene42RWNormsFormat;
import org.apache.lucene.util.LuceneTestCase;
/**
@@ -50,7 +50,14 @@ public class Lucene45RWCodec extends Luc
return fieldInfosFormat;
}
- private static final NormsFormat norms = new Lucene42NormsFormat();
+ private static final DocValuesFormat docValues = new Lucene45RWDocValuesFormat();
+
+ @Override
+ public DocValuesFormat getDocValuesFormatForField(String field) {
+ return docValues;
+ }
+
+ private static final NormsFormat norms = new Lucene42RWNormsFormat();
@Override
public NormsFormat normsFormat() {
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene46/Lucene46RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene46/Lucene46RWCodec.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene46/Lucene46RWCodec.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene46/Lucene46RWCodec.java Mon Jun 16 19:26:19 2014
@@ -17,8 +17,10 @@ package org.apache.lucene.codecs.lucene4
* limitations under the License.
*/
+import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
+import org.apache.lucene.codecs.lucene42.Lucene42RWNormsFormat;
+import org.apache.lucene.codecs.lucene45.Lucene45RWDocValuesFormat;
/**
* Read-write version of {@link Lucene46Codec} for testing.
@@ -26,7 +28,14 @@ import org.apache.lucene.codecs.lucene42
@SuppressWarnings("deprecation")
public class Lucene46RWCodec extends Lucene46Codec {
- private static final NormsFormat norms = new Lucene42NormsFormat();
+ private static final DocValuesFormat docValues = new Lucene45RWDocValuesFormat();
+
+ @Override
+ public DocValuesFormat getDocValuesFormatForField(String field) {
+ return docValues;
+ }
+
+ private static final NormsFormat norms = new Lucene42RWNormsFormat();
@Override
public NormsFormat normsFormat() {
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java Mon Jun 16 19:26:19 2014
@@ -512,6 +512,41 @@ public class AssertingAtomicReader exten
}
/** Wraps a SortedSetDocValues but with additional asserts */
+ public static class AssertingSortedNumericDocValues extends SortedNumericDocValues {
+ private final SortedNumericDocValues in;
+ private final int maxDoc;
+
+ public AssertingSortedNumericDocValues(SortedNumericDocValues in, int maxDoc) {
+ this.in = in;
+ this.maxDoc = maxDoc;
+ }
+
+ @Override
+ public void setDocument(int doc) {
+ assert doc >= 0 && doc < maxDoc;
+ in.setDocument(doc);
+ // check the values are actually sorted
+ long previous = Long.MIN_VALUE;
+ for (int i = 0; i < in.count(); i++) {
+ long v = in.valueAt(i);
+ assert v >= previous;
+ previous = v;
+ }
+ }
+
+ @Override
+ public long valueAt(int index) {
+ assert index < in.count();
+ return in.valueAt(index);
+ }
+
+ @Override
+ public int count() {
+ return in.count();
+ }
+ }
+
+ /** Wraps a SortedSetDocValues but with additional asserts */
public static class AssertingSortedSetDocValues extends SortedSetDocValues {
private final SortedSetDocValues in;
private final int maxDoc;
@@ -608,6 +643,20 @@ public class AssertingAtomicReader exten
return null;
}
}
+
+ @Override
+ public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException {
+ SortedNumericDocValues dv = super.getSortedNumericDocValues(field);
+ FieldInfo fi = getFieldInfos().fieldInfo(field);
+ if (dv != null) {
+ assert fi != null;
+ assert fi.getDocValuesType() == FieldInfo.DocValuesType.SORTED_NUMERIC;
+ return new AssertingSortedNumericDocValues(dv, maxDoc());
+ } else {
+ assert fi == null || fi.getDocValuesType() != FieldInfo.DocValuesType.SORTED_NUMERIC;
+ return null;
+ }
+ }
@Override
public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {