You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/10/26 11:28:53 UTC
svn commit: r1027415 - in
/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene: index/
index/codecs/ index/codecs/docvalues/ index/values/ util/
Author: simonw
Date: Tue Oct 26 09:28:52 2010
New Revision: 1027415
URL: http://svn.apache.org/viewvc?rev=1027415&view=rev
Log:
LUCENE-2700: added missing files
Added:
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesCodec.java (with props)
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java (with props)
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java (with props)
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java (with props)
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/Pair.java (with props)
Modified:
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1027415&r1=1027414&r2=1027415&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Tue Oct 26 09:28:52 2010
@@ -17,24 +17,15 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.index.codecs.FieldsConsumer;
-import org.apache.lucene.index.values.Ints;
-import org.apache.lucene.index.values.Floats;
-import org.apache.lucene.index.values.Bytes;
-import org.apache.lucene.index.values.ValuesAttribute;
-import org.apache.lucene.index.values.Writer;
-import org.apache.lucene.index.values.codec.DocValuesConsumer;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.FloatsRef;
-import org.apache.lucene.util.LongsRef;
-
import java.io.IOException;
import java.util.Collection;
-import java.util.Comparator;
-import java.util.Map;
import java.util.HashMap;
+import java.util.Map;
+import org.apache.lucene.index.codecs.FieldsConsumer;
+import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
+import org.apache.lucene.index.values.ValuesAttribute;
+import org.apache.lucene.store.Directory;
/**
* This is a DocConsumer that gathers all fields under the
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java?rev=1027415&r1=1027414&r2=1027415&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java Tue Oct 26 09:28:52 2010
@@ -24,8 +24,8 @@ import java.io.IOException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
import org.apache.lucene.index.values.ValuesAttribute;
-import org.apache.lucene.index.values.codec.DocValuesConsumer;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.RamUsageEstimator;
@@ -407,4 +407,4 @@ final class DocFieldProcessorPerThread e
}
}
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java?rev=1027415&r1=1027414&r2=1027415&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java Tue Oct 26 09:28:52 2010
@@ -22,10 +22,9 @@ import org.apache.lucene.index.Fields;
import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.index.values.Writer;
-import org.apache.lucene.index.values.Values;
-import org.apache.lucene.index.values.codec.DocValuesConsumer;
import java.io.IOException;
import java.io.Closeable;
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java?rev=1027415&r1=1027414&r2=1027415&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java Tue Oct 26 09:28:52 2010
@@ -23,7 +23,6 @@ import java.util.Set;
import java.util.HashSet;
import java.util.Iterator;
import java.util.IdentityHashMap;
-import java.util.TreeMap;
import java.util.TreeSet;
import java.io.IOException;
@@ -35,8 +34,8 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
import org.apache.lucene.index.values.DocValues;
-import org.apache.lucene.index.values.codec.DocValuesConsumer;
import org.apache.lucene.store.Directory;
Added: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesCodec.java?rev=1027415&view=auto
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesCodec.java (added)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesCodec.java Tue Oct 26 09:28:52 2010
@@ -0,0 +1,298 @@
+package org.apache.lucene.index.codecs.docvalues;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.FieldsConsumer;
+import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.TermsConsumer;
+import org.apache.lucene.index.values.DocValues;
+import org.apache.lucene.index.values.Writer;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.AttributeSource;
+
+/**
+ * A codec that adds DocValues support to a given codec transparently.
+ */
+public class DocValuesCodec extends Codec {
+ private final Map<String, WrappingFieldsConsumer> consumers = new HashMap<String, WrappingFieldsConsumer>();
+ private final Codec other;
+
+ public DocValuesCodec(Codec other) {
+ this.name = "docvalues_" + other.name;
+ this.other = other;
+ }
+
+ @Override
+ public FieldsConsumer fieldsConsumer(SegmentWriteState state)
+ throws IOException {
+ WrappingFieldsConsumer consumer;
+ if ((consumer = consumers.get(state.segmentName)) == null) {
+ consumer = new WrappingFieldsConsumer(other);
+ }
+ consumer.state = state; // nocommit this is a hack and only necessary since
+ // we want to initialized the wrapped
+ // fieldsConsumer lazily with a SegmentWriteState created after the docvalue
+ // ones is. We should fix this in DocumentWriter I guess. See
+ // DocFieldProcessor too!
+ return consumer;
+ }
+
+ private static class WrappingFieldsConsumer extends FieldsConsumer {
+ SegmentWriteState state;
+ private final List<DocValuesConsumer> docValuesConsumers = new ArrayList<DocValuesConsumer>();
+ private FieldsConsumer wrappedConsumer;
+ private final Codec other;
+
+ public WrappingFieldsConsumer(Codec other) {
+ this.other = other;
+ }
+
+ @Override
+ public void close() throws IOException {
+ synchronized (this) {
+ if (wrappedConsumer != null)
+ wrappedConsumer.close();
+ }
+ }
+
+ @Override
+ public synchronized DocValuesConsumer addValuesField(FieldInfo field)
+ throws IOException {
+ DocValuesConsumer consumer = DocValuesConsumer.create(state.segmentName,
+ state.directory, field, null); // TODO: set comparator here
+ docValuesConsumers.add(consumer);
+ return consumer;
+ }
+
+ @Override
+ public TermsConsumer addField(FieldInfo field) throws IOException {
+ synchronized (this) {
+ if (wrappedConsumer == null)
+ wrappedConsumer = other.fieldsConsumer(state);
+ }
+ return wrappedConsumer.addField(field);
+ }
+ }
+
+ @Override
+ public FieldsProducer fieldsProducer(SegmentReadState state)
+ throws IOException {
+ Directory dir = state.dir;
+ Set<String> files = new HashSet<String>();
+
+ other.files(dir, state.segmentInfo, files);
+ for (String string : files) {
+ if (dir.fileExists(string))
+ return new WrappingFielsdProducer(state.segmentInfo, state.dir,
+ state.fieldInfos, other.fieldsProducer(state));
+ }
+ return new WrappingFielsdProducer(state.segmentInfo, state.dir,
+ state.fieldInfos, FieldsProducer.EMPTY);
+
+ }
+
+ @Override
+ public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files)
+ throws IOException {
+ Set<String> otherFiles = new HashSet<String>();
+ other.files(dir, segmentInfo, otherFiles);
+ for (String string : otherFiles) {
+ if (dir.fileExists(string))
+ files.add(string);
+ }
+
+ for (String file : dir.listAll()) {
+ if (file.startsWith(segmentInfo.name)
+ && (file.endsWith(Writer.DATA_EXTENSION) || file
+ .endsWith(Writer.INDEX_EXTENSION))) {
+ files.add(file);
+ }
+ }
+ // files.add(IndexFileNames.segmentFileName(segmentInfo.name, "",
+ // Writer.DATA_EXTENSION));
+ // files.add(IndexFileNames.segmentFileName(segmentInfo.name, "",
+ // Writer.INDEX_EXTENSION));
+
+ }
+
+ @Override
+ public void getExtensions(Set<String> extensions) {
+ other.getExtensions(extensions);
+ extensions.add(Writer.DATA_EXTENSION);
+ extensions.add(Writer.INDEX_EXTENSION);
+ }
+
+ static class WrappingFielsdProducer extends DocValuesProducerBase {
+
+ private final FieldsProducer other;
+
+ WrappingFielsdProducer(SegmentInfo si, Directory dir, FieldInfos fieldInfo,
+ FieldsProducer other) throws IOException {
+ super(si, dir, fieldInfo);
+ this.other = other;
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ other.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public void loadTermsIndex(int indexDivisor) throws IOException {
+ other.loadTermsIndex(indexDivisor);
+ }
+
+ @Override
+ public FieldsEnum iterator() throws IOException {
+ return new WrappingFieldsEnum(other.iterator(), docValues.entrySet()
+ .iterator());
+ }
+
+ @Override
+ public Terms terms(String field) throws IOException {
+ return other.terms(field);
+ }
+ }
+
+ static abstract class NameValue<V> {
+ String name;
+ V value;
+
+ NameValue<?> smaller(NameValue<?> other) throws IOException {
+ if (other.name == null) {
+ if (this.name == null) {
+ return null;
+ }
+ return this;
+ } else if (this.name == null) {
+ return other;
+ }
+ final int res = this.name.compareTo(other.name);
+ if (res < 0)
+ return this;
+ if (res == 0)
+ other.name = this.name;
+ return other;
+ }
+
+ abstract NameValue<V> next() throws IOException;
+ }
+
+ static class FieldsEnumNameValue extends NameValue<FieldsEnum> {
+ @Override
+ NameValue<FieldsEnum> next() throws IOException {
+ name = value.next();
+ return this;
+ }
+
+ }
+
+ static class DocValueNameValue extends NameValue<DocValues> {
+ Iterator<Entry<String, DocValues>> iter;
+
+ @Override
+ NameValue<DocValues> next() {
+ if (iter.hasNext()) {
+ Entry<String, DocValues> next = iter.next();
+ value = next.getValue();
+ name = next.getKey();
+ } else {
+ name = null;
+ }
+ return this;
+ }
+
+ }
+
+ static class WrappingFieldsEnum extends FieldsEnum {
+ private final DocValueNameValue docValues = new DocValueNameValue();
+ private final NameValue<FieldsEnum> fieldsEnum = new FieldsEnumNameValue();
+ private NameValue<?> coordinator;
+
+ @Override
+ public AttributeSource attributes() {
+ return fieldsEnum.value.attributes();
+ }
+
+ public WrappingFieldsEnum(FieldsEnum wrapped,
+ Iterator<Entry<String, DocValues>> docValues) {
+ this.docValues.iter = docValues;
+ this.fieldsEnum.value = wrapped;
+ coordinator = null;
+
+ }
+
+ @Override
+ public DocValues docValues() throws IOException {
+ if (docValues.name == coordinator.name)
+ return docValues.value;
+ return null;
+ }
+
+ @Override
+ public String next() throws IOException {
+ if (coordinator == null) {
+ coordinator = fieldsEnum.next().smaller(docValues.next());
+ // old = coordinator.name;
+ } else {
+ String current = coordinator.name;
+ if (current == docValues.name) {
+ docValues.next();
+ }
+ if (current == fieldsEnum.name) {
+ fieldsEnum.next();
+ }
+ coordinator = docValues.smaller(fieldsEnum);
+
+ }
+ return coordinator == null ? null : coordinator.name;
+
+ }
+
+ @Override
+ public TermsEnum terms() throws IOException {
+ if (fieldsEnum.name == coordinator.name)
+ return fieldsEnum.value.terms();
+ return null;
+ }
+
+ }
+
+}
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesCodec.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesCodec.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java?rev=1027415&view=auto
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java (added)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java Tue Oct 26 09:28:52 2010
@@ -0,0 +1,67 @@
+package org.apache.lucene.index.codecs.docvalues;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.values.DocValues;
+import org.apache.lucene.index.values.ValuesAttribute;
+import org.apache.lucene.index.values.Writer;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+public abstract class DocValuesConsumer {
+ public abstract void add(int docID, ValuesAttribute attr) throws IOException;
+
+ public abstract void finish(int docCount) throws IOException;
+
+ public abstract void files(Collection<String> files) throws IOException;
+
+ public void merge(List<MergeState> states) throws IOException {
+ for (MergeState state : states) {
+ merge(state);
+ }
+ }
+
+ protected abstract void merge(MergeState mergeState) throws IOException;
+
+
+ public static class MergeState {
+ public final DocValues reader;
+ public final int docBase;
+ public final int docCount;
+ public final Bits bits;
+
+ public MergeState(DocValues reader, int docBase, int docCount, Bits bits) {
+ assert reader != null;
+ this.reader = reader;
+ this.docBase = docBase;
+ this.docCount = docCount;
+ this.bits = bits;
+ }
+ }
+
+ public static DocValuesConsumer create(String segmentName, Directory directory,
+ FieldInfo field, Comparator<BytesRef> comp) throws IOException {
+ final String id = segmentName + "_" + field.number;
+ return Writer.create(field.getIndexValues(), id, directory, comp);
+ }
+}
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java?rev=1027415&view=auto
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java (added)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java Tue Oct 26 09:28:52 2010
@@ -0,0 +1,99 @@
+package org.apache.lucene.index.codecs.docvalues;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.Collection;
+import java.util.TreeMap;
+
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.values.Bytes;
+import org.apache.lucene.index.values.DocValues;
+import org.apache.lucene.index.values.Floats;
+import org.apache.lucene.index.values.Ints;
+import org.apache.lucene.index.values.Values;
+import org.apache.lucene.index.values.Writer;
+import org.apache.lucene.store.Directory;
+
+public abstract class DocValuesProducerBase extends FieldsProducer{
+
+ protected final TreeMap<String, DocValues> docValues = new TreeMap<String, DocValues>();
+
+ protected DocValuesProducerBase(SegmentInfo si, Directory dir, FieldInfos fieldInfo) throws IOException {
+ load(fieldInfo, si.name, si.docCount, dir);
+ }
+
+ @Override
+ public DocValues docValues(String field) throws IOException {
+ return docValues.get(field);
+ }
+
+ // Only opens files... doesn't actually load any values
+ protected void load(FieldInfos fieldInfos, String segment, int docCount,
+ Directory dir) throws IOException {
+ final int numFields = fieldInfos.size();
+ for (int i = 0; i < numFields; i++) {
+ final FieldInfo fieldInfo = fieldInfos.fieldInfo(i);
+ final Values v = fieldInfo.getIndexValues();
+ final String field = fieldInfo.name;
+ final String id = IndexFileNames.segmentFileName(segment, Integer
+ .toString(fieldInfo.number), "");
+ if (v != null && dir.fileExists(id + "." + Writer.DATA_EXTENSION)) {
+ docValues.put(field, loadDocValues(docCount, dir, id, v));
+ }
+ }
+ }
+
+ protected DocValues loadDocValues(int docCount, Directory dir, String id,
+ Values v) throws IOException {
+ switch (v) {
+ case PACKED_INTS:
+ return Ints.getValues(dir, id, false);
+ case PACKED_INTS_FIXED:
+ return Ints.getValues(dir, id, true);
+ case SIMPLE_FLOAT_4BYTE:
+ return Floats.getValues(dir, id, docCount);
+ case SIMPLE_FLOAT_8BYTE:
+ return Floats.getValues(dir, id, docCount);
+ case BYTES_FIXED_STRAIGHT:
+ return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount);
+ case BYTES_FIXED_DEREF:
+ return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount);
+ case BYTES_FIXED_SORTED:
+ return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount);
+ case BYTES_VAR_STRAIGHT:
+ return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount);
+ case BYTES_VAR_DEREF:
+ return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount);
+ case BYTES_VAR_SORTED:
+ return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount);
+ default:
+ throw new IllegalStateException("unrecognized index values mode " + v);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ Collection<DocValues> values = docValues.values();
+ for (DocValues docValues : values) {
+ docValues.close();
+ }
+ }
+}
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java?rev=1027415&view=auto
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java (added)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java Tue Oct 26 09:28:52 2010
@@ -0,0 +1,300 @@
+package org.apache.lucene.index.values;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.MultiTermsEnum.TermsEnumIndex;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FloatsRef;
+import org.apache.lucene.util.LongsRef;
+import org.apache.lucene.util.ReaderUtil.Slice;
+
+public class MultiDocValues extends DocValues {
+
+ public static class DocValuesIndex {
+ public final static DocValuesIndex[] EMPTY_ARRAY = new DocValuesIndex[0];
+ final int subIndex;
+ final DocValues docValues;
+
+ public DocValuesIndex(DocValues docValues, int subIndex) {
+ this.docValues = docValues;
+ this.subIndex = subIndex;
+ }
+ }
+
+ private DocValuesIndex[] docValuesIdx;
+ private Slice[] subSlices;
+
+ public MultiDocValues(Slice[] subSlices) {
+ this.subSlices = subSlices;
+ }
+
+ public MultiDocValues(DocValuesIndex[] docValuesIdx, Slice[] subSlices) {
+ this(subSlices);
+ reset(docValuesIdx);
+ }
+
+ @Override
+ public ValuesEnum getEnum(AttributeSource source) throws IOException {
+ return new MultiValuesEnum(subSlices, docValuesIdx, docValuesIdx[0].docValues.type());
+ }
+
+ @Override
+ public Source load() throws IOException {
+ return new MultiSource(subSlices, docValuesIdx);
+ }
+
+ public void close() throws IOException {
+ //
+ }
+
+ public DocValues reset(DocValuesIndex[] docValuesIdx) {
+ this.docValuesIdx = docValuesIdx;
+ return this;
+ }
+
+ private static class MultiValuesEnum extends ValuesEnum {
+ private int numDocs_ = 0;
+ private int pos = -1;
+ private int start = 0;
+ private ValuesEnum current;
+ private Slice[] subSlices;
+ private DocValuesIndex[] docValuesIdx;
+ private final int maxDoc;
+
+ public MultiValuesEnum(Slice[] subSlices, DocValuesIndex[] docValuesIdx, Values type) {
+ super(type);
+ this.subSlices = subSlices;
+ this.docValuesIdx = docValuesIdx;
+ Slice slice = subSlices[subSlices.length-1];
+ maxDoc = slice.start + slice.length;
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+// int n = target - start;
+// do {
+// if (target >= maxDoc)
+// return pos = NO_MORE_DOCS;
+// if (n >= numDocs_) {
+// int idx = readerIndex(target);
+// if (enumCache[idx] == null) {
+// try {
+// DocValues indexValues = subReaders[idx].docValues(id);
+// if (indexValues != null) // nocommit does that work with default
+// // values?
+// enumCache[idx] = indexValues.getEnum(this.attributes());
+// else
+// enumCache[idx] = new DummyEnum(this.attributes(),
+// subSlices[idx].length, attr.type());
+// } catch (IOException ex) {
+// // nocommit what to do here?
+// throw new RuntimeException(ex);
+// }
+// }
+// current = enumCache[idx];
+// start = subSlices[idx].start;
+// numDocs_ = subSlices[idx].length;
+// n = target - start;
+// }
+// target = start + numDocs_;
+// } while ((n = current.advance(n)) == NO_MORE_DOCS);
+ return pos = start + current.docID();
+ }
+
+ @Override
+ public int docID() {
+ return pos;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return advance(pos + 1);
+ }
+ }
+
+ private class MultiSource extends Source {
+ private int numDocs_ = 0;
+ private int start = 0;
+ private Source current;
+ private Slice[] subSlices;
+ private DocValuesIndex[] docVAluesIdx;
+
+ public MultiSource(Slice[] subSlices, DocValuesIndex[] docValuesIdx) {
+ this.subSlices = subSlices;
+ this.docVAluesIdx = docValuesIdx;
+ }
+
+ public long ints(int docID) {
+// int n = docID - start;
+// if (n >= numDocs_) {
+// int idx = readerIndex(docID);
+// try {
+// current = subReaders[idx].getIndexValuesCache().getInts(id);
+// if (current == null) // nocommit does that work with default values?
+// current = new DummySource();
+// } catch (IOException ex) {
+// // nocommit what to do here?
+// throw new RuntimeException(ex);
+// }
+// start = starts[idx];
+// numDocs_ = subReaders[idx].maxDoc();
+// n = docID - start;
+// }
+// return current.ints(n);
+ return 0l;
+ }
+
+ public double floats(int docID) {
+// int n = docID - start;
+// if (n >= numDocs_) {
+// int idx = readerIndex(docID);
+// try {
+// current = subReaders[idx].getIndexValuesCache().getFloats(id);
+// if (current == null) // nocommit does that work with default values?
+// current = new DummySource();
+// } catch (IOException ex) {
+// // nocommit what to do here?
+// throw new RuntimeException(ex);
+// }
+// numDocs_ = subReaders[idx].maxDoc();
+//
+// start = starts[idx];
+// n = docID - start;
+// }
+// return current.floats(n);
+ return 0d;
+ }
+
+ public BytesRef bytes(int docID) {
+// int n = docID - start;
+// if (n >= numDocs_) {
+// int idx = readerIndex(docID);
+// try {
+// current = subReaders[idx].getIndexValuesCache().getBytes(id);
+// if (current == null) // nocommit does that work with default values?
+// current = new DummySource();
+// } catch (IOException ex) {
+// // nocommit what to do here?
+// throw new RuntimeException(ex);
+// }
+// numDocs_ = subReaders[idx].maxDoc();
+// start = starts[idx];
+// n = docID - start;
+// }
+// return current.bytes(n);
+ return null;
+ }
+
+ public long ramBytesUsed() {
+ return current.ramBytesUsed();
+ }
+
+ }
+
+ private static class DummySource extends Source {
+ private final BytesRef ref = new BytesRef();
+
+ @Override
+ public BytesRef bytes(int docID) {
+ return ref;
+ }
+
+ @Override
+ public double floats(int docID) {
+ return 0.0d;
+ }
+
+ @Override
+ public long ints(int docID) {
+ return 0;
+ }
+
+ public long ramBytesUsed() {
+ return 0;
+ }
+ }
+
+ private static class DummyEnum extends ValuesEnum {
+ private int pos = -1;
+ private final int maxDoc;
+
+ public DummyEnum(AttributeSource source, int maxDoc, Values type) {
+ super(source, type);
+ this.maxDoc = maxDoc;
+ switch (type) {
+ case BYTES_VAR_STRAIGHT:
+ case BYTES_FIXED_STRAIGHT:
+ case BYTES_FIXED_DEREF:
+ case BYTES_FIXED_SORTED:
+ case BYTES_VAR_DEREF:
+ case BYTES_VAR_SORTED:
+ // nocommit - this is not correct for Fixed_straight
+ BytesRef bytes = attr.bytes();
+ bytes.length = 0;
+ bytes.offset = 0;
+ break;
+ case PACKED_INTS:
+ case PACKED_INTS_FIXED:
+ LongsRef ints = attr.ints();
+ ints.set(0);
+ break;
+
+ case SIMPLE_FLOAT_4BYTE:
+ case SIMPLE_FLOAT_8BYTE:
+ FloatsRef floats = attr.floats();
+ floats.set(0d);
+ break;
+ default:
+ throw new IllegalArgumentException("unknown Values type: " + type);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ return pos = (pos < maxDoc ? target : NO_MORE_DOCS);
+ }
+
+ @Override
+ public int docID() {
+ return pos;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return advance(pos + 1);
+ }
+ }
+
+ @Override
+ public Values type() {
+ return this.docValuesIdx[0].docValues.type();
+ }
+
+}
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/Pair.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/Pair.java?rev=1027415&view=auto
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/Pair.java (added)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/Pair.java Tue Oct 26 09:28:52 2010
@@ -0,0 +1,36 @@
+package org.apache.lucene.util;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Simple Pair
+ * @lucene.internal
+ */
+public class Pair<Cur,Cud> {
+ public final Cur cur;
+ public final Cud cud;
+
+ /**
+ * Create a simple pair
+ * @param cur the first element
+ * @param cud the second element
+ */
+ public Pair(Cur cur, Cud cud) {
+ this.cur = cur;
+ this.cud = cud;
+ }
+}
\ No newline at end of file
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/Pair.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/Pair.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL