You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/10/30 12:56:01 UTC
svn commit: r1029017 - in /lucene/dev/branches/docvalues/lucene/src:
java/org/apache/lucene/index/ java/org/apache/lucene/index/codecs/
java/org/apache/lucene/index/values/ java/org/apache/lucene/search/
test/org/apache/lucene/index/values/
Author: simonw
Date: Sat Oct 30 10:56:01 2010
New Revision: 1029017
URL: http://svn.apache.org/viewvc?rev=1029017&view=rev
Log:
LUCENE-2700: Enabled MultiField tests & introduced SourceCache
Added:
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/SourceCache.java (with props)
Modified:
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/ValuesAttributeImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java?rev=1029017&r1=1029016&r2=1029017&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java Sat Oct 30 10:56:01 2010
@@ -163,10 +163,9 @@ public final class MultiFieldsEnum exte
FieldsEnumWithSlice withSlice = enumWithSlices[i];
Slice slice = withSlice.slice;
final DocValues values = withSlice.fields.docValues();
-
final int start = slice.start;
final int length = slice.length;
- if (values != null) {
+ if (values != null && currentField.equals(withSlice.current)) {
if (docsUpto != start) {
type = values.type();
docValuesIndex.add(new MultiDocValues.DocValuesIndex(
@@ -176,9 +175,8 @@ public final class MultiFieldsEnum exte
docValuesIndex.add(new MultiDocValues.DocValuesIndex(values, start,
length));
docsUpto = start + length;
-
- } else if (i+1 == numEnums && !docValuesIndex.isEmpty()) {
+ } else if (i + 1 == numEnums && !docValuesIndex.isEmpty()) {
docValuesIndex.add(new MultiDocValues.DocValuesIndex(
new MultiDocValues.DummyDocValues(start, type), docsUpto, start
- docsUpto));
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java?rev=1029017&r1=1029016&r2=1029017&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java Sat Oct 30 10:56:01 2010
@@ -67,7 +67,11 @@ public abstract class FieldsConsumer imp
}
if (mergeState.fieldInfo.hasDocValues()) {
final DocValues docValues = fieldsEnum.docValues();
- assert docValues != null : "DocValues are null for " + mergeState.fieldInfo.getDocValues();
+ // TODO: is this assert values and if so when?
+// assert docValues != null : "DocValues are null for " + mergeState.fieldInfo.getDocValues();
+ if(docValues == null) { // for now just continue
+ continue;
+ }
final DocValuesConsumer docValuesConsumer = addValuesField(mergeState.fieldInfo);
assert docValuesConsumer != null;
docValuesConsumer.merge(mergeState, docValues);
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java?rev=1029017&r1=1029016&r2=1029017&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java Sat Oct 30 10:56:01 2010
@@ -25,11 +25,8 @@ import org.apache.lucene.util.BytesRef;
public abstract class DocValues implements Closeable {
- private final Object lock = new Object();
-
- private Source cachedReference;
-
public static final DocValues[] EMPTY_ARRAY = new DocValues[0];
+ private SourceCache cache = new SourceCache.DirectSourceCache();
public ValuesEnum getEnum() throws IOException {
return getEnum(null);
@@ -40,20 +37,12 @@ public abstract class DocValues implemen
public abstract Source load() throws IOException;
- public Source getCached(boolean load) throws IOException {
- synchronized (lock) { // TODO make sorted source cachable too
- if (load && cachedReference == null)
- cachedReference = load();
- return cachedReference;
- }
+ public Source getSource() throws IOException {
+ return cache.load(this);
}
-
- public Source releaseCached() {
- synchronized (lock) {
- final Source retVal = cachedReference;
- cachedReference = null;
- return retVal;
- }
+
+ public SortedSource getSortedSorted(Comparator<BytesRef> comparator) throws IOException {
+ return cache.laodSorted(this, comparator);
}
public SortedSource loadSorted(Comparator<BytesRef> comparator)
@@ -64,7 +53,14 @@ public abstract class DocValues implemen
public abstract Values type();
public void close() throws IOException {
- releaseCached();
+ this.cache.close(this);
+ }
+
+ public void setCache(SourceCache cache) {
+ synchronized (this.cache) {
+ this.cache.close(this);
+ this.cache = cache;
+ }
}
/**
@@ -137,5 +133,5 @@ public abstract class DocValues implemen
*/
public abstract LookupResult getByValue(BytesRef value);
}
-
+
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java?rev=1029017&r1=1029016&r2=1029017&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java Sat Oct 30 10:56:01 2010
@@ -19,7 +19,6 @@ package org.apache.lucene.index.values;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.lucene.index.values.DocValues.Source;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FloatsRef;
@@ -28,7 +27,7 @@ import org.apache.lucene.util.ReaderUtil
public class MultiDocValues extends DocValues {
- public static class DocValuesIndex { // nocommit is this necessary?
+ public static class DocValuesIndex {
public final static DocValuesIndex[] EMPTY_ARRAY = new DocValuesIndex[0];
final int start;
final int length;
@@ -96,17 +95,7 @@ public class MultiDocValues extends DocV
public Source load() throws IOException {
return DUMMY;
}
-
- @Override
- public Source getCached(boolean load) throws IOException {
- return DUMMY;
- }
-
- @Override
- public Source releaseCached() {
- return DUMMY;
- }
-
+
@Override
public Values type() {
return type;
Added: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/SourceCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/SourceCache.java?rev=1029017&view=auto
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/SourceCache.java (added)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/SourceCache.java Sat Oct 30 10:56:01 2010
@@ -0,0 +1,65 @@
+package org.apache.lucene.index.values;
+
+import java.io.IOException;
+import java.util.Comparator;
+
+import org.apache.lucene.index.values.DocValues.SortedSource;
+import org.apache.lucene.index.values.DocValues.Source;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Per {@link DocValues} {@link Source} cache.
+ */
+public abstract class SourceCache {
+ public abstract Source load(DocValues values) throws IOException;
+
+ public abstract SortedSource laodSorted(DocValues values,
+ Comparator<BytesRef> comp) throws IOException;
+
+ public abstract void invalidate(DocValues values);
+
+ public synchronized void close(DocValues values) {
+ invalidate(values);
+ }
+
+ public static final class DirectSourceCache extends SourceCache {
+ private Source ref;
+ private SortedSource sortedRef;
+
+ public synchronized Source load(DocValues values) throws IOException {
+ if (ref == null)
+ ref = values.load();
+ return ref;
+ }
+
+ public synchronized SortedSource laodSorted(DocValues values,
+ Comparator<BytesRef> comp) throws IOException {
+ if (sortedRef == null)
+ sortedRef = values.loadSorted(comp);
+ return sortedRef;
+ }
+
+ public synchronized void invalidate(DocValues values) {
+ ref = null;
+ sortedRef = null;
+ }
+ }
+
+}
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/SourceCache.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/SourceCache.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/ValuesAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/ValuesAttributeImpl.java?rev=1029017&r1=1029016&r2=1029017&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/ValuesAttributeImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/ValuesAttributeImpl.java Sat Oct 30 10:56:01 2010
@@ -62,7 +62,11 @@ public class ValuesAttributeImpl extends
@Override
public void clear() {
- // TODO
+ bytes = null;
+ ints = null;
+ floats = null;
+ type = null;
+ bytesComp = null;
}
@Override
@@ -91,9 +95,6 @@ public class ValuesAttributeImpl extends
}
}
- /* (non-Javadoc)
- * @see java.lang.Object#hashCode()
- */
@Override
public int hashCode() {
final int prime = 31;
@@ -105,9 +106,6 @@ public class ValuesAttributeImpl extends
return result;
}
- /* (non-Javadoc)
- * @see java.lang.Object#equals(java.lang.Object)
- */
@Override
public boolean equals(Object obj) {
if (this == obj)
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=1029017&r1=1029016&r2=1029017&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java Sat Oct 30 10:56:01 2010
@@ -353,7 +353,7 @@ public abstract class FieldComparator {
@Override
public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException {
- currentReaderValues = reader.docValues(field).getCached(true);
+ currentReaderValues = reader.docValues(field).getSource();
return this;
}
@@ -555,7 +555,7 @@ public abstract class FieldComparator {
@Override
public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException {
- currentReaderValues = reader.docValues(field).getCached(true);
+ currentReaderValues = reader.docValues(field).getSource();
return this;
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java?rev=1029017&r1=1029016&r2=1029017&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java Sat Oct 30 10:56:01 2010
@@ -44,9 +44,9 @@ import org.apache.lucene.index.MergePoli
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.index.codecs.docvalues.DocValuesCodec;
import org.apache.lucene.index.values.DocValues.SortedSource;
import org.apache.lucene.index.values.DocValues.Source;
-import org.apache.lucene.index.codecs.docvalues.DocValuesCodec;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.BytesRef;
@@ -56,9 +56,7 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util._TestUtil;
-import org.junit.After;
import org.junit.AfterClass;
-import org.junit.Before;
import org.junit.BeforeClass;
public class TestIndexValues extends LuceneTestCase {
@@ -162,7 +160,7 @@ public class TestIndexValues extends Luc
Source s;
DocValues.SortedSource ss;
if (mode == Bytes.Mode.SORTED) {
- s = ss = r.loadSorted(comp);
+ s = ss = getSortedSource(r, comp);
} else {
s = getSource(r);
ss = null;
@@ -436,13 +434,16 @@ public class TestIndexValues extends Luc
switch (val) {
case PACKED_INTS:
case PACKED_INTS_FIXED: {
+ if(val == Values.PACKED_INTS_FIXED)
+ getDocValues(r, val.name());
DocValues intsReader = getDocValues(r, val.name());
Source ints = getSource(intsReader);
+
ValuesEnum intsEnum = intsReader.getEnum();
assertNotNull(intsEnum);
LongsRef enumRef = intsEnum.addAttribute(ValuesAttribute.class).ints();
for (int i = 0; i < base; i++) {
- assertEquals(0, ints.getInt(i));
+ assertEquals("index " + i, 0, ints.getInt(i));
assertEquals(val.name() + " base: " + base + " index: " + i, i,
random.nextBoolean() ? intsEnum.advance(i) : intsEnum.nextDoc());
assertEquals(0, enumRef.get());
@@ -586,8 +587,7 @@ public class TestIndexValues extends Luc
// TODO test unoptimized with deletions
if (withDeletions || random.nextBoolean())
- ;
- w.optimize();
+ w.optimize();
return deleted;
}
@@ -598,7 +598,7 @@ public class TestIndexValues extends Luc
final List<Values> byteVariantList = new ArrayList<Values>(BYTES);
// run in random order to test if fill works correctly during merges
Collections.shuffle(byteVariantList, random);
- final int numValues = 333 + random.nextInt(150);
+ final int numValues = 179 + random.nextInt(151);
for (Values byteIndexValue : byteVariantList) {
List<Closeable> closeables = new ArrayList<Closeable>();
@@ -713,7 +713,11 @@ public class TestIndexValues extends Luc
}
private Source getSource(DocValues values) throws IOException {
- return random.nextBoolean() ? values.load() : values.getCached(true);
+ // getSource uses cache internally
+ return random.nextBoolean() ? values.load() : values.getSource();
+ }
+ private SortedSource getSortedSource(DocValues values, Comparator<BytesRef> comparator) throws IOException {
+ // getSortedSource uses cache internally
+ return random.nextBoolean() ? values.loadSorted(comparator) : values.getSortedSorted(comparator);
}
-
}