You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/06/16 15:36:55 UTC
svn commit: r1602864 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/
lucene/codecs/src/resources/META-INF/services/
lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/ lucene/co...
Author: rmuir
Date: Mon Jun 16 13:36:54 2014
New Revision: 1602864
URL: http://svn.apache.org/r1602864
Log:
LUCENE-5761: Remove DiskDocValuesFormat
Removed:
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/
lucene/dev/branches/branch_4x/lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java
lucene/dev/branches/branch_4x/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSortedSetSortFieldSelectors.java
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1602864&r1=1602863&r2=1602864&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Mon Jun 16 13:36:54 2014
@@ -135,6 +135,9 @@ API Changes
* LUCENE-5757: Moved RamUsageEstimator's reflection-based processing to RamUsageTester
in the test-framework module. (Robert Muir)
+* LUCENE-5761: Removed DiskDocValuesFormat, it was very inefficient and saved very little
+ RAM over the default codec. (Robert Muir)
+
Optimizations
* LUCENE-5603: hunspell stemmer more efficiently strips prefixes
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat?rev=1602864&r1=1602863&r2=1602864&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat Mon Jun 16 13:36:54 2014
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.diskdv.DiskDocValuesFormat
org.apache.lucene.codecs.memory.MemoryDocValuesFormat
org.apache.lucene.codecs.memory.DirectDocValuesFormat
org.apache.lucene.codecs.simpletext.SimpleTextDocValuesFormat
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java?rev=1602864&r1=1602863&r2=1602864&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java Mon Jun 16 13:36:54 2014
@@ -65,7 +65,7 @@ import org.apache.lucene.util.packed.Mon
import org.apache.lucene.util.packed.PackedInts;
/** reader for {@link Lucene45DocValuesFormat} */
-public class Lucene45DocValuesProducer extends DocValuesProducer implements Closeable {
+class Lucene45DocValuesProducer extends DocValuesProducer implements Closeable {
private final Map<Integer,NumericEntry> numerics;
private final Map<Integer,BinaryEntry> binaries;
private final Map<Integer,SortedSetEntry> sortedSets;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java?rev=1602864&r1=1602863&r2=1602864&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java Mon Jun 16 13:36:54 2014
@@ -39,7 +39,7 @@ import org.apache.lucene.util.packed.Mon
import org.apache.lucene.util.packed.PackedInts;
/** writer for {@link Lucene49DocValuesFormat} */
-public class Lucene49DocValuesConsumer extends DocValuesConsumer implements Closeable {
+class Lucene49DocValuesConsumer extends DocValuesConsumer implements Closeable {
static final int BLOCK_SIZE = 16384;
static final int ADDRESS_INTERVAL = 16;
@@ -299,7 +299,7 @@ public class Lucene49DocValuesConsumer e
}
/** expert: writes a value dictionary for a sorted/sortedset field */
- protected void addTermsDict(FieldInfo field, final Iterable<BytesRef> values) throws IOException {
+ private void addTermsDict(FieldInfo field, final Iterable<BytesRef> values) throws IOException {
// first check if its a "fixed-length" terms dict
int minLength = Integer.MAX_VALUE;
int maxLength = Integer.MIN_VALUE;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java?rev=1602864&r1=1602863&r2=1602864&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java Mon Jun 16 13:36:54 2014
@@ -64,7 +64,7 @@ import org.apache.lucene.util.packed.Dir
import org.apache.lucene.util.packed.MonotonicBlockPackedReader;
/** reader for {@link Lucene49DocValuesFormat} */
-public class Lucene49DocValuesProducer extends DocValuesProducer implements Closeable {
+class Lucene49DocValuesProducer extends DocValuesProducer implements Closeable {
private final Map<Integer,NumericEntry> numerics;
private final Map<Integer,BinaryEntry> binaries;
private final Map<Integer,SortedSetEntry> sortedSets;
@@ -81,7 +81,7 @@ public class Lucene49DocValuesProducer e
private final Map<Integer,MonotonicBlockPackedReader> ordIndexInstances = new HashMap<>();
/** expert: instantiates a new reader */
- protected Lucene49DocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
+ Lucene49DocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
// read in the entries from the metadata file.
ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
@@ -409,9 +409,8 @@ public class Lucene49DocValuesProducer e
};
}
- /** returns an address instance for variable-length binary values.
- * @lucene.internal */
- protected MonotonicBlockPackedReader getAddressInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException {
+ /** returns an address instance for variable-length binary values. */
+ private MonotonicBlockPackedReader getAddressInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException {
final MonotonicBlockPackedReader addresses;
synchronized (addressInstances) {
MonotonicBlockPackedReader addrInstance = addressInstances.get(field.number);
@@ -451,9 +450,8 @@ public class Lucene49DocValuesProducer e
};
}
- /** returns an address instance for prefix-compressed binary values.
- * @lucene.internal */
- protected MonotonicBlockPackedReader getIntervalInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException {
+ /** returns an address instance for prefix-compressed binary values. */
+ private MonotonicBlockPackedReader getIntervalInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException {
final MonotonicBlockPackedReader addresses;
final long interval = bytes.addressInterval;
synchronized (addressInstances) {
@@ -528,9 +526,8 @@ public class Lucene49DocValuesProducer e
};
}
- /** returns an address instance for sortedset ordinal lists
- * @lucene.internal */
- protected MonotonicBlockPackedReader getOrdIndexInstance(IndexInput data, FieldInfo field, NumericEntry entry) throws IOException {
+ /** returns an address instance for sortedset ordinal lists */
+ private MonotonicBlockPackedReader getOrdIndexInstance(IndexInput data, FieldInfo field, NumericEntry entry) throws IOException {
final MonotonicBlockPackedReader ordIndex;
synchronized (ordIndexInstances) {
MonotonicBlockPackedReader ordIndexInstance = ordIndexInstances.get(field.number);
@@ -712,7 +709,7 @@ public class Lucene49DocValuesProducer e
}
/** metadata entry for a numeric docvalues field */
- protected static class NumericEntry {
+ static class NumericEntry {
private NumericEntry() {}
/** offset to the bitset representing docsWithField, or -1 if no documents have missing values */
long missingOffset;
@@ -737,7 +734,7 @@ public class Lucene49DocValuesProducer e
}
/** metadata entry for a binary docvalues field */
- protected static class BinaryEntry {
+ static class BinaryEntry {
private BinaryEntry() {}
/** offset to the bitset representing docsWithField, or -1 if no documents have missing values */
long missingOffset;
@@ -760,7 +757,7 @@ public class Lucene49DocValuesProducer e
}
/** metadata entry for a sorted-set docvalues field */
- protected static class SortedSetEntry {
+ static class SortedSetEntry {
private SortedSetEntry() {}
int format;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java?rev=1602864&r1=1602863&r2=1602864&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java Mon Jun 16 13:36:54 2014
@@ -97,7 +97,7 @@ import org.apache.lucene.util.packed.Pac
* </ol>
* @lucene.experimental
*/
-public class Lucene49NormsFormat extends NormsFormat {
+public final class Lucene49NormsFormat extends NormsFormat {
/** Sole Constructor */
public Lucene49NormsFormat() {}
Modified: lucene/dev/branches/branch_4x/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSortedSetSortFieldSelectors.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSortedSetSortFieldSelectors.java?rev=1602864&r1=1602863&r2=1602864&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSortedSetSortFieldSelectors.java (original)
+++ lucene/dev/branches/branch_4x/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSortedSetSortFieldSelectors.java Mon Jun 16 13:36:54 2014
@@ -18,7 +18,6 @@ package org.apache.lucene.sandbox.querie
*/
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.diskdv.DiskDocValuesFormat;
import org.apache.lucene.codecs.lucene49.Lucene49DocValuesFormat;
import org.apache.lucene.codecs.memory.DirectDocValuesFormat;
import org.apache.lucene.document.Document;
@@ -48,10 +47,9 @@ public class TestSortedSetSortFieldSelec
public static void beforeClass() throws Exception {
savedCodec = Codec.getDefault();
// currently only these codecs that support random access ordinals
- int victim = random().nextInt(3);
+ int victim = random().nextInt(2);
switch(victim) {
case 0: Codec.setDefault(TestUtil.alwaysDocValuesFormat(new DirectDocValuesFormat()));
- case 1: Codec.setDefault(TestUtil.alwaysDocValuesFormat(new DiskDocValuesFormat()));
default: Codec.setDefault(TestUtil.alwaysDocValuesFormat(new Lucene49DocValuesFormat()));
}
}
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java?rev=1602864&r1=1602863&r2=1602864&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java Mon Jun 16 13:36:54 2014
@@ -23,12 +23,12 @@ import org.apache.lucene.codecs.NormsFor
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.diskdv.DiskDocValuesFormat;
-import org.apache.lucene.codecs.diskdv.DiskNormsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat;
import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
import org.apache.lucene.codecs.lucene49.Lucene49Codec;
+import org.apache.lucene.codecs.lucene49.Lucene49DocValuesFormat;
+import org.apache.lucene.codecs.lucene49.Lucene49NormsFormat;
/** Codec that tries to use as little ram as possible because he spent all his money on beer */
// TODO: better name :)
@@ -40,9 +40,8 @@ public class CheapBastardCodec extends F
// uncompressing versions, waste lots of disk but no ram
private final StoredFieldsFormat storedFields = new Lucene40StoredFieldsFormat();
private final TermVectorsFormat termVectors = new Lucene40TermVectorsFormat();
- // these go to disk for all docvalues/norms datastructures
- private final DocValuesFormat docValues = new DiskDocValuesFormat();
- private final NormsFormat norms = new DiskNormsFormat();
+ private final DocValuesFormat docValues = new Lucene49DocValuesFormat();
+ private final NormsFormat norms = new Lucene49NormsFormat();
public CheapBastardCodec() {
super("CheapBastard", new Lucene49Codec());
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java?rev=1602864&r1=1602863&r2=1602864&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java Mon Jun 16 13:36:54 2014
@@ -32,7 +32,6 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat;
import org.apache.lucene.codecs.asserting.AssertingPostingsFormat;
import org.apache.lucene.codecs.bloom.TestBloomFilteredLucene41Postings;
-import org.apache.lucene.codecs.diskdv.DiskDocValuesFormat;
import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
import org.apache.lucene.codecs.lucene41ords.Lucene41WithOrds;
import org.apache.lucene.codecs.lucene49.Lucene49Codec;
@@ -153,7 +152,6 @@ public class RandomCodec extends Lucene4
addDocValues(avoidCodecs,
new Lucene49DocValuesFormat(),
- new DiskDocValuesFormat(),
new MemoryDocValuesFormat(),
new SimpleTextDocValuesFormat(),
new AssertingDocValuesFormat());
Modified: lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml?rev=1602864&r1=1602863&r2=1602864&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml Mon Jun 16 13:36:54 2014
@@ -21,8 +21,8 @@
<fieldType name="string_simpletext" class="solr.StrField" postingsFormat="SimpleText"/>
<fieldType name="string_standard" class="solr.StrField" postingsFormat="Lucene41"/>
- <fieldType name="string_disk" class="solr.StrField" docValuesFormat="Disk" />
- <fieldType name="string_memory" class="solr.StrField" docValuesFormat="Lucene49" />
+ <fieldType name="string_disk" class="solr.StrField" docValuesFormat="Lucene49" />
+ <fieldType name="string_memory" class="solr.StrField" docValuesFormat="Memory" />
<fieldType name="string" class="solr.StrField" />
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java?rev=1602864&r1=1602863&r2=1602864&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java Mon Jun 16 13:36:54 2014
@@ -53,9 +53,9 @@ public class TestCodecSupport extends So
Map<String, SchemaField> fields = h.getCore().getLatestSchema().getFields();
SchemaField schemaField = fields.get("string_disk_f");
PerFieldDocValuesFormat format = (PerFieldDocValuesFormat) codec.docValuesFormat();
- assertEquals("Disk", format.getDocValuesFormatForField(schemaField.getName()).getName());
+ assertEquals("Lucene49", format.getDocValuesFormatForField(schemaField.getName()).getName());
schemaField = fields.get("string_memory_f");
- assertEquals("Lucene49",
+ assertEquals("Memory",
format.getDocValuesFormatForField(schemaField.getName()).getName());
schemaField = fields.get("string_f");
assertEquals("Lucene49",
@@ -78,9 +78,9 @@ public class TestCodecSupport extends So
Codec codec = h.getCore().getCodec();
PerFieldDocValuesFormat format = (PerFieldDocValuesFormat) codec.docValuesFormat();
- assertEquals("Disk", format.getDocValuesFormatForField("foo_disk").getName());
- assertEquals("Disk", format.getDocValuesFormatForField("bar_disk").getName());
- assertEquals("Lucene49", format.getDocValuesFormatForField("foo_memory").getName());
- assertEquals("Lucene49", format.getDocValuesFormatForField("bar_memory").getName());
+ assertEquals("Lucene49", format.getDocValuesFormatForField("foo_disk").getName());
+ assertEquals("Lucene49", format.getDocValuesFormatForField("bar_disk").getName());
+ assertEquals("Memory", format.getDocValuesFormatForField("foo_memory").getName());
+ assertEquals("Memory", format.getDocValuesFormatForField("bar_memory").getName());
}
}