You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/01/23 19:21:37 UTC
svn commit: r1437608 - in /lucene/dev/branches/lucene4547/lucene:
core/src/java/org/apache/lucene/codecs/lucene40/
core/src/java/org/apache/lucene/codecs/lucene41/
test-framework/src/java/org/apache/lucene/codecs/lucene40/
test-framework/src/java/org/a...
Author: rmuir
Date: Wed Jan 23 18:21:37 2013
New Revision: 1437608
URL: http://svn.apache.org/viewvc?rev=1437608&view=rev
Log:
more 4.0 back compat
Added:
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java (with props)
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40LyingDocValuesFormat.java
- copied, changed from r1437575, lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesFormat.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java (with props)
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40LyingRWDocValuesFormat.java (with props)
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWDocValuesFormat.java (with props)
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWNormsFormat.java (with props)
Modified:
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java?rev=1437608&r1=1437607&r2=1437608&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java Wed Jan 23 18:21:37 2013
@@ -87,19 +87,17 @@ public class Lucene40Codec extends Codec
return infosFormat;
}
- //nocommit need a read-only Lucene40DocValuesFormat / read-write in the impersonator
- private final DocValuesFormat defaultDVFormat = new Lucene40DocValuesFormat();
+ private final DocValuesFormat defaultDVFormat = new Lucene40LyingDocValuesFormat();
@Override
- public final DocValuesFormat docValuesFormat() {
+ public DocValuesFormat docValuesFormat() {
return defaultDVFormat;
}
- // nocommit need a read-only Lucene40NormsFormat / read-write in the impersonator
private final NormsFormat normsFormat = new Lucene40NormsFormat();
@Override
- public final NormsFormat normsFormat() {
+ public NormsFormat normsFormat() {
return normsFormat;
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesFormat.java?rev=1437608&r1=1437607&r2=1437608&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesFormat.java Wed Jan 23 18:21:37 2013
@@ -20,52 +20,38 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosReader.LegacyDocValuesType;
-import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
-import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.BytesRef;
-
-// nocommit: still a lie, but allows javadocs @links to work
-// nocommit: make read-only and move to impersonator
-public class Lucene40DocValuesFormat extends Lucene42DocValuesFormat {
+// NOTE: not registered in SPI, doesnt respect segment suffix, etc
+// for back compat only!
+public class Lucene40DocValuesFormat extends DocValuesFormat {
+
+ public Lucene40DocValuesFormat() {
+ super("Lucene40");
+ }
+
@Override
public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- final DocValuesConsumer delegate = super.fieldsConsumer(state);
- return new DocValuesConsumer() {
-
- @Override
- public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
- // hack: here we would examine the numerics and simulate in the impersonator the best we can
- // e.g. if they are all in byte/int range write fixed, otherwise write packed or whatever
- field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.VAR_INTS.name());
- delegate.addNumericField(field, values);
- }
-
- @Override
- public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
- field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.BYTES_VAR_STRAIGHT.name());
- delegate.addBinaryField(field, values);
- }
-
- @Override
- public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
- field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.BYTES_VAR_SORTED.name());
- delegate.addSortedField(field, values, docToOrd);
- }
-
- @Override
- public void close() throws IOException {
- delegate.close();
- }
- };
+ throw new UnsupportedOperationException("this codec can only be used for reading");
}
-
+
@Override
public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
- return super.fieldsProducer(state);
+ String filename = IndexFileNames.segmentFileName(state.segmentInfo.name,
+ "dv",
+ IndexFileNames.COMPOUND_FILE_EXTENSION);
+ return new Lucene40DocValuesReader(state, filename, Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY);
}
+
+ // constants for VAR_INTS
+ static final String VAR_INTS_CODEC_NAME = "PackedInts";
+ static final int VAR_INTS_VERSION_START = 0;
+ static final int VAR_INTS_VERSION_CURRENT = VAR_INTS_VERSION_START;
+
+ static final byte VAR_INTS_PACKED = 0x00;
+ static final byte VAR_INTS_FIXED_64 = 0x01;
}
Added: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java?rev=1437608&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java (added)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java Wed Jan 23 18:21:37 2013
@@ -0,0 +1,137 @@
+package org.apache.lucene.codecs.lucene40;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosReader.LegacyDocValuesType;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.store.CompoundFileDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.packed.PackedInts;
+
+class Lucene40DocValuesReader extends DocValuesProducer {
+ private final Directory dir;
+ private final SegmentReadState state;
+ private final String legacyKey;
+
+ // ram instances we have already loaded
+ private final Map<Integer,NumericDocValues> numericInstances =
+ new HashMap<Integer,NumericDocValues>();
+ private final Map<Integer,BinaryDocValues> binaryInstances =
+ new HashMap<Integer,BinaryDocValues>();
+ private final Map<Integer,SortedDocValues> sortedInstances =
+ new HashMap<Integer,SortedDocValues>();
+
+ Lucene40DocValuesReader(SegmentReadState state, String filename, String legacyKey) throws IOException {
+ this.state = state;
+ this.legacyKey = legacyKey;
+ this.dir = new CompoundFileDirectory(state.directory, filename, state.context, false);
+ }
+
+ @Override
+ public synchronized NumericDocValues getNumeric(FieldInfo field) throws IOException {
+ NumericDocValues instance = numericInstances.get(field.number);
+ if (instance == null) {
+ switch(LegacyDocValuesType.valueOf(field.getAttribute(legacyKey))) {
+ case VAR_INTS:
+ instance = loadVarIntsField(field);
+ break;
+ default:
+ throw new AssertionError(); // nocommit, implement the other types
+ }
+ numericInstances.put(field.number, instance);
+ }
+ return instance;
+ }
+
+ private NumericDocValues loadVarIntsField(FieldInfo field) throws IOException {
+ String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, Integer.toString(field.number), "dat");
+ IndexInput input = dir.openInput(fileName, state.context);
+ boolean success = false;
+ try {
+ CodecUtil.checkHeader(input, Lucene40DocValuesFormat.VAR_INTS_CODEC_NAME,
+ Lucene40DocValuesFormat.VAR_INTS_VERSION_START,
+ Lucene40DocValuesFormat.VAR_INTS_VERSION_CURRENT);
+ byte header = input.readByte();
+ if (header == Lucene40DocValuesFormat.VAR_INTS_FIXED_64) {
+ int maxDoc = state.segmentInfo.getDocCount();
+ final long values[] = new long[maxDoc];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = input.readLong();
+ }
+ return new NumericDocValues() {
+ @Override
+ public long get(int docID) {
+ return values[docID];
+ }
+ };
+ } else if (header == Lucene40DocValuesFormat.VAR_INTS_PACKED) {
+ final long minValue = input.readLong();
+ final long defaultValue = input.readLong();
+ final PackedInts.Reader reader = PackedInts.getReader(input);
+ return new NumericDocValues() {
+ @Override
+ public long get(int docID) {
+ final long value = reader.get(docID);
+ if (value == defaultValue) {
+ return 0;
+ } else {
+ return minValue + value;
+ }
+ }
+ };
+ } else {
+ throw new CorruptIndexException("invalid VAR_INTS header byte: " + header + " (resource=" + input + ")");
+ }
+ } finally {
+ if (success) {
+ IOUtils.close(input);
+ } else {
+ IOUtils.closeWhileHandlingException(input);
+ }
+ }
+ }
+
+ @Override
+ public synchronized BinaryDocValues getBinary(FieldInfo field) throws IOException {
+ throw new AssertionError();
+ }
+
+ @Override
+ public synchronized SortedDocValues getSorted(FieldInfo field) throws IOException {
+ throw new AssertionError();
+ }
+
+ @Override
+ public void close() throws IOException {
+ dir.close();
+ }
+}
Copied: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40LyingDocValuesFormat.java (from r1437575, lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesFormat.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40LyingDocValuesFormat.java?p2=lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40LyingDocValuesFormat.java&p1=lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesFormat.java&r1=1437575&r2=1437608&rev=1437608&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40LyingDocValuesFormat.java Wed Jan 23 18:21:37 2013
@@ -21,47 +21,17 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosReader.LegacyDocValuesType;
import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
-import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.BytesRef;
// nocommit: still a lie, but allows javadocs @links to work
// nocommit: make read-only and move to impersonator
-public class Lucene40DocValuesFormat extends Lucene42DocValuesFormat {
+public class Lucene40LyingDocValuesFormat extends Lucene42DocValuesFormat {
@Override
public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- final DocValuesConsumer delegate = super.fieldsConsumer(state);
- return new DocValuesConsumer() {
-
- @Override
- public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
- // hack: here we would examine the numerics and simulate in the impersonator the best we can
- // e.g. if they are all in byte/int range write fixed, otherwise write packed or whatever
- field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.VAR_INTS.name());
- delegate.addNumericField(field, values);
- }
-
- @Override
- public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
- field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.BYTES_VAR_STRAIGHT.name());
- delegate.addBinaryField(field, values);
- }
-
- @Override
- public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
- field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.BYTES_VAR_SORTED.name());
- delegate.addSortedField(field, values, docToOrd);
- }
-
- @Override
- public void close() throws IOException {
- delegate.close();
- }
- };
+ throw new UnsupportedOperationException("this codec can only be used for reading");
}
@Override
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java?rev=1437608&r1=1437607&r2=1437608&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java Wed Jan 23 18:21:37 2013
@@ -21,49 +21,23 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosReader.LegacyDocValuesType;
-import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
-import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.BytesRef;
-// nocommit: still a lie, but allows javadocs @links to work
-// nocommit: make read-only and move to impersonator
-public class Lucene40NormsFormat extends Lucene42NormsFormat {
-
+public class Lucene40NormsFormat extends NormsFormat {
+
@Override
public DocValuesConsumer normsConsumer(SegmentWriteState state) throws IOException {
- final DocValuesConsumer delegate = super.normsConsumer(state);
- return new DocValuesConsumer() {
-
- @Override
- public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
- // hack: here we would examine the numerics and simulate in the impersonator the best we can
- // e.g. if they are all in byte/int range write fixed, otherwise write packed or whatever
- field.putAttribute(Lucene40FieldInfosReader.LEGACY_NORM_TYPE_KEY, LegacyDocValuesType.VAR_INTS.name());
- delegate.addNumericField(field, values);
- }
-
- @Override
- public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
- assert false;
- }
-
- @Override
- public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
- assert false;
- }
-
- @Override
- public void close() throws IOException {
- delegate.close();
- }
- };
+ throw new UnsupportedOperationException("this codec can only be used for reading");
}
@Override
public DocValuesProducer normsProducer(SegmentReadState state) throws IOException {
- return super.normsProducer(state);
+ String filename = IndexFileNames.segmentFileName(state.segmentInfo.name,
+ "nrm",
+ IndexFileNames.COMPOUND_FILE_EXTENSION);
+ return new Lucene40DocValuesReader(state, filename, Lucene40FieldInfosReader.LEGACY_NORM_TYPE_KEY);
}
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java?rev=1437608&r1=1437607&r2=1437608&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java Wed Jan 23 18:21:37 2013
@@ -32,13 +32,12 @@ import org.apache.lucene.codecs.StoredFi
import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat;
import org.apache.lucene.codecs.compressing.CompressionMode;
-import org.apache.lucene.codecs.lucene40.Lucene40DocValuesFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40LyingDocValuesFormat;
import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosFormat;
import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40NormsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat;
import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
@@ -121,18 +120,16 @@ public class Lucene41Codec extends Codec
}
@Override
- public final DocValuesFormat docValuesFormat() {
+ public DocValuesFormat docValuesFormat() {
return dvFormat;
}
private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41");
- // nocommit need a read-only Lucene40DocValuesFormat / read-write in the impersonator
- private final DocValuesFormat dvFormat = new Lucene40DocValuesFormat();
- // nocommit need a read-only Lucene40NormsFormat / read-write in the impersonator
+ private final DocValuesFormat dvFormat = new Lucene40LyingDocValuesFormat();
private final NormsFormat normsFormat = new Lucene40NormsFormat();
@Override
- public final NormsFormat normsFormat() {
+ public NormsFormat normsFormat() {
return normsFormat;
}
}
Added: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java?rev=1437608&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java (added)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java Wed Jan 23 18:21:37 2013
@@ -0,0 +1,119 @@
+package org.apache.lucene.codecs.lucene40;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosReader.LegacyDocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.CompoundFileDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.packed.PackedInts;
+
+class Lucene40DocValuesWriter extends DocValuesConsumer {
+ private final Directory dir;
+ private final SegmentWriteState state;
+ private final String legacyKey;
+
+ // note: intentionally ignores seg suffix
+ // String filename = IndexFileNames.segmentFileName(state.segmentInfo.name, "dv", IndexFileNames.COMPOUND_FILE_EXTENSION);
+ Lucene40DocValuesWriter(SegmentWriteState state, String filename, String legacyKey) throws IOException {
+ this.state = state;
+ this.legacyKey = legacyKey;
+ this.dir = new CompoundFileDirectory(state.directory, filename, state.context, true);
+ }
+
+ @Override
+ public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
+ // TODO: examine the values: and simulate all the possibilities.
+ // e.g. if all values fit in a byte, write a fixed_8 etc.
+ field.putAttribute(legacyKey, LegacyDocValuesType.VAR_INTS.name());
+ String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, Integer.toString(field.number), "dat");
+ IndexOutput data = dir.createOutput(fileName, state.context);
+ boolean success = false;
+ try {
+ addVarIntsField(data, values);
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(data);
+ } else {
+ IOUtils.closeWhileHandlingException(data);
+ }
+ }
+ }
+
+ @Override
+ public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
+ assert false;
+ }
+
+ @Override
+ public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
+ assert false;
+ }
+
+ @Override
+ public void close() throws IOException {
+ dir.close();
+ }
+
+ private void addVarIntsField(IndexOutput output, Iterable<Number> values) throws IOException {
+ long minValue = Long.MAX_VALUE;
+ long maxValue = Long.MIN_VALUE;
+ for (Number n : values) {
+ long v = n.longValue();
+ minValue = Math.min(minValue, v);
+ maxValue = Math.max(maxValue, v);
+ }
+
+ CodecUtil.writeHeader(output,
+ Lucene40DocValuesFormat.VAR_INTS_CODEC_NAME,
+ Lucene40DocValuesFormat.VAR_INTS_VERSION_CURRENT);
+
+ final long delta = maxValue - minValue;
+
+ if (delta < 0) {
+ // writes longs
+ output.writeByte(Lucene40DocValuesFormat.VAR_INTS_FIXED_64);
+ for (Number n : values) {
+ output.writeLong(n.longValue());
+ }
+ } else {
+ // writes packed ints
+ output.writeByte(Lucene40DocValuesFormat.VAR_INTS_PACKED);
+ output.writeLong(minValue);
+ output.writeLong(0 - minValue); // default value (representation of 0)
+ PackedInts.Writer writer = PackedInts.getWriter(output,
+ state.segmentInfo.getDocCount(),
+ PackedInts.bitsRequired(delta),
+ PackedInts.DEFAULT);
+ for (Number n : values) {
+ writer.add(n.longValue() - minValue);
+ }
+ writer.finish();
+ }
+ }
+}
Added: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40LyingRWDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40LyingRWDocValuesFormat.java?rev=1437608&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40LyingRWDocValuesFormat.java (added)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40LyingRWDocValuesFormat.java Wed Jan 23 18:21:37 2013
@@ -0,0 +1,65 @@
+package org.apache.lucene.codecs.lucene40;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosReader.LegacyDocValuesType;
+import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.util.BytesRef;
+
+public class Lucene40LyingRWDocValuesFormat extends Lucene40LyingDocValuesFormat {
+ private final DocValuesFormat lie = new Lucene42DocValuesFormat();
+
+ // nocommit: a lie
+ @Override
+ public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+ final DocValuesConsumer delegate = lie.fieldsConsumer(state);
+ return new DocValuesConsumer() {
+
+ @Override
+ public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
+ // hack: here we would examine the numerics and simulate in the impersonator the best we can
+ // e.g. if they are all in byte/int range write fixed, otherwise write packed or whatever
+ field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.VAR_INTS.name());
+ delegate.addNumericField(field, values);
+ }
+
+ @Override
+ public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
+ field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.BYTES_VAR_STRAIGHT.name());
+ delegate.addBinaryField(field, values);
+ }
+
+ @Override
+ public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
+ field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.BYTES_VAR_SORTED.name());
+ delegate.addSortedField(field, values, docToOrd);
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ }
+ };
+ }
+}
Modified: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java?rev=1437608&r1=1437607&r2=1437608&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java (original)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java Wed Jan 23 18:21:37 2013
@@ -2,8 +2,10 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
+import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FieldInfosWriter;
+import org.apache.lucene.codecs.NormsFormat;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -31,8 +33,21 @@ public final class Lucene40RWCodec exten
}
};
+ private final DocValuesFormat docValues = new Lucene40LyingRWDocValuesFormat();
+ private final NormsFormat norms = new Lucene40RWNormsFormat();
+
@Override
public FieldInfosFormat fieldInfosFormat() {
return fieldInfos;
}
+
+ @Override
+ public DocValuesFormat docValuesFormat() {
+ return docValues;
+ }
+
+ @Override
+ public NormsFormat normsFormat() {
+ return norms;
+ }
}
Added: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWDocValuesFormat.java?rev=1437608&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWDocValuesFormat.java (added)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWDocValuesFormat.java Wed Jan 23 18:21:37 2013
@@ -0,0 +1,35 @@
+package org.apache.lucene.codecs.lucene40;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentWriteState;
+
+public class Lucene40RWDocValuesFormat extends Lucene40DocValuesFormat {
+
+ @Override
+ public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+ String filename = IndexFileNames.segmentFileName(state.segmentInfo.name,
+ "dv",
+ IndexFileNames.COMPOUND_FILE_EXTENSION);
+ return new Lucene40DocValuesWriter(state, filename, Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY);
+ }
+}
Added: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWNormsFormat.java?rev=1437608&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWNormsFormat.java (added)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWNormsFormat.java Wed Jan 23 18:21:37 2013
@@ -0,0 +1,35 @@
+package org.apache.lucene.codecs.lucene40;
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentWriteState;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class Lucene40RWNormsFormat extends Lucene40NormsFormat {
+
+ @Override
+ public DocValuesConsumer normsConsumer(SegmentWriteState state) throws IOException {
+ String filename = IndexFileNames.segmentFileName(state.segmentInfo.name,
+ "nrm",
+ IndexFileNames.COMPOUND_FILE_EXTENSION);
+ return new Lucene40DocValuesWriter(state, filename, Lucene40FieldInfosReader.LEGACY_NORM_TYPE_KEY);
+ }
+}
Modified: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java?rev=1437608&r1=1437607&r2=1437608&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java (original)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java Wed Jan 23 18:21:37 2013
@@ -2,11 +2,15 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
+import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FieldInfosWriter;
+import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosFormat;
import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosWriter;
+import org.apache.lucene.codecs.lucene40.Lucene40LyingRWDocValuesFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40RWNormsFormat;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -37,6 +41,9 @@ public class Lucene41RWCodec extends Luc
}
};
+ private final DocValuesFormat docValues = new Lucene40LyingRWDocValuesFormat();
+ private final NormsFormat norms = new Lucene40RWNormsFormat();
+
@Override
public FieldInfosFormat fieldInfosFormat() {
return fieldInfos;
@@ -46,4 +53,14 @@ public class Lucene41RWCodec extends Luc
public StoredFieldsFormat storedFieldsFormat() {
return fieldsFormat;
}
+
+ @Override
+ public DocValuesFormat docValuesFormat() {
+ return docValues;
+ }
+
+ @Override
+ public NormsFormat normsFormat() {
+ return norms;
+ }
}