You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by bi...@apache.org on 2011/11/16 19:05:09 UTC
svn commit: r1202816 - in /incubator/accumulo/trunk: ./
src/core/src/main/java/org/apache/accumulo/core/iterators/
src/core/src/main/java/org/apache/accumulo/core/iterators/user/
src/core/src/test/java/org/apache/accumulo/core/iterators/
Author: billie
Date: Wed Nov 16 18:05:09 2011
New Revision: 1202816
URL: http://svn.apache.org/viewvc?rev=1202816&view=rev
Log:
ACCUMULO-148 ACCUMULO-151 merged changes to trunk
Modified:
incubator/accumulo/trunk/ (props changed)
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/AgeOffFilter.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ColumnAgeOffFilter.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/NoVisFilter.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/TimestampFilter.java
incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/CombinerTest.java
Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 16 18:05:09 2011
@@ -1 +1,2 @@
/incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383
+/incubator/accumulo/branches/1.4:1202767-1202802,1202815
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java?rev=1202816&r1=1202815&r2=1202816&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java Wed Nov 16 18:05:09 2011
@@ -37,17 +37,33 @@ import org.apache.accumulo.core.iterator
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
-@SuppressWarnings("deprecation")
+/*
+ * A SortedKeyValueIterator that combines the Values for different versions of a Key into a single Value.
+ * Combiner will replace one or more versions of a Key and their Values with the most recent Key and a Value which is the result of the reduce method.
+ *
+ * Subclasses must implement a reduce method:
+ * public Value reduce(Key key, Iterator<Value> iter);
+ *
+ * This reduce method will be passed the most recent Key and an iterator over the Values for all non-deleted versions of that Key.
+ */
public abstract class Combiner extends WrappingIterator implements OptionDescriber {
static final Logger log = Logger.getLogger(Combiner.class);
public static final String COLUMN_PREFIX = "column:";
+ /*
+ * A Java Iterator that iterates over the Values for a given Key from a source SortedKeyValueIterator.
+ */
public static class ValueIterator implements Iterator<Value> {
Key topKey;
SortedKeyValueIterator<Key,Value> source;
boolean hasNext;
- ValueIterator(SortedKeyValueIterator<Key,Value> source) {
+ /*
+ * Constructs an iterator over Values whose Keys are versions of the current topKey of the source SortedKeyValueIterator.
+ *
+ * @param source The SortedKeyValueIterator<Key,Value> from which to read data.
+ */
+ public ValueIterator(SortedKeyValueIterator<Key,Value> source) {
this.source = source;
topKey = source.getTopKey();
hasNext = _hasNext();
@@ -57,11 +73,21 @@ public abstract class Combiner extends W
return source.hasTop() && !source.getTopKey().isDeleted() && topKey.equals(source.getTopKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS);
}
+ /*
+ * @return <tt>true</tt> if there is another Value
+ *
+ * @see java.util.Iterator#hasNext()
+ */
@Override
public boolean hasNext() {
return hasNext;
}
+ /*
+ * @return the next Value
+ *
+ * @see java.util.Iterator#next()
+ */
@Override
public Value next() {
if (!hasNext)
@@ -76,6 +102,11 @@ public abstract class Combiner extends W
return topValue;
}
+ /*
+ * unsupported
+ *
+ * @see java.util.Iterator#remove()
+ */
@Override
public void remove() {
throw new UnsupportedOperationException();
@@ -120,11 +151,16 @@ public abstract class Combiner extends W
private Key workKey = new Key();
+ /*
+ * Sets the topKey and topValue based on the top key of the source. If the column of the source top key is in the set of combiners, topKey will be the top key
+ * of the source and topValue will be the result of the reduce method. Otherwise, topKey and topValue will be unchanged. (They are always set to null before
+ * this method is called.)
+ */
private void findTop() throws IOException {
// check if aggregation is needed
if (super.hasTop()) {
workKey.set(super.getTopKey());
- if (combiners.isEmpty() || combiners.contains(workKey)) {
+ if (combiners.contains(workKey)) {
if (workKey.isDeleted())
return;
topKey = workKey;
@@ -138,8 +174,7 @@ public abstract class Combiner extends W
@Override
public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
- // do not want to seek to the middle of a value that should be
- // combined...
+ // do not want to seek to the middle of a value that should be combined...
Range seekRange = IteratorUtil.maximizeStartKeyTimeStamp(range);
@@ -149,8 +184,7 @@ public abstract class Combiner extends W
if (range.getStartKey() != null) {
while (hasTop() && getTopKey().equals(range.getStartKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS)
&& getTopKey().getTimestamp() > range.getStartKey().getTimestamp()) {
- // the value has a more recent time stamp, so
- // pass it up
+ // the value has a more recent time stamp, so pass it up
// log.debug("skipping "+getTopKey());
next();
}
@@ -161,6 +195,15 @@ public abstract class Combiner extends W
}
}
+ /*
+ * Reduces a list of Values into a single Value.
+ *
+ * @param key The most recent version of the Key being reduced.
+ *
+ * @param iter An iterator over the Values for different versions of the key.
+ *
+ * @return The combined Value.
+ */
public abstract Value reduce(Key key, Iterator<Value> iter);
private ColumnSet combiners;
@@ -188,6 +231,15 @@ public abstract class Combiner extends W
return true;
}
+ /*
+ * Adds a column (colf and colq) to an IteratorSetting.
+ *
+ * @param colf The column family.
+ *
+ * @param colq The column qualifier (<tt>null</tt> if unspecified).
+ *
+ * @param is The IteratorSetting to which to add the column parameter.
+ */
public static void addColumn(Text colf, Text colq, IteratorSetting is) {
String column = PerColumnIteratorConfig.encodeColumns(colf, colq);
is.addOption(COLUMN_PREFIX + column, "");
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java?rev=1202816&r1=1202815&r2=1202816&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java Wed Nov 16 18:05:09 2011
@@ -27,6 +27,17 @@ import org.apache.accumulo.core.data.Key
import org.apache.accumulo.core.data.Value;
import org.apache.hadoop.io.WritableUtils;
+/*
+ * A TypedValueCombiner that translates each Value to a Long before reducing, then encodes the reduced Long back to a Value.
+ *
+ * Subclasses must implement a typedReduce method:
+ * public Long typedReduce(Key key, Iterator<Long> iter);
+ *
+ * This typedReduce method will be passed the most recent Key and an iterator over the Values (translated to Longs) for all non-deleted versions of that Key.
+ *
+ * A required option for this Combiner is "type" which indicates which type of Encoder to use to encode and decode Longs into Values. Supported types are
+ * VARNUM, LONG, and STRING which indicate the VarNumEncoder, LongEncoder, and StringEncoder respectively.
+ */
public abstract class LongCombiner extends TypedValueCombiner<Long> {
public static final String TYPE = "type";
@@ -65,6 +76,7 @@ public abstract class LongCombiner exten
@Override
public boolean validateOptions(Map<String,String> options) {
+ super.validateOptions(options);
if (options.get(TYPE) == null)
return false;
try {
@@ -75,6 +87,9 @@ public abstract class LongCombiner exten
return true;
}
+ /*
+ * An Encoder that uses a variable-length encoding for Longs. It uses WritableUtils.writeVLong and WritableUtils.readVLong for encoding and decoding.
+ */
public static class VarNumEncoder implements Encoder<Long> {
@Override
public byte[] encode(Long v) {
@@ -101,6 +116,9 @@ public abstract class LongCombiner exten
}
}
+ /*
+ * An Encoder that uses an 8-byte encoding for Longs.
+ */
public static class LongEncoder implements Encoder<Long> {
@Override
public byte[] encode(Long l) {
@@ -129,6 +147,9 @@ public abstract class LongCombiner exten
}
}
+ /*
+ * An Encoder that uses a String representation of Longs. It uses Long.toString and Long.parseLong for encoding and decoding.
+ */
public static class StringEncoder implements Encoder<Long> {
@Override
public byte[] encode(Long v) {
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java?rev=1202816&r1=1202815&r2=1202816&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java Wed Nov 16 18:05:09 2011
@@ -23,16 +23,32 @@ import org.apache.accumulo.core.data.Key
import org.apache.accumulo.core.data.Value;
/*
- * Subclasses should implement a switch on the "type" variable
- * in their reduce method.
+ * A Combiner that decodes each Value to type V before reducing, then encodes the result of typedReduce back to Value.
+ *
+ * Subclasses must implement a typedReduce method:
+ * public V typedReduce(Key key, Iterator<V> iter);
+ *
+ * This typedReduce method will be passed the most recent Key and an iterator over the Values (translated to Vs) for all non-deleted versions of that Key.
+ *
+ * Subclasses may implement a switch on the "type" variable to choose an Encoder in their init method.
*/
public abstract class TypedValueCombiner<V> extends Combiner {
protected Encoder<V> encoder;
+ /*
+ * A Java Iterator that translates an Iterator<Value> to an Iterator<V> using the decode method of an Encoder.
+ */
private static class VIterator<V> implements Iterator<V> {
private Iterator<Value> source;
private Encoder<V> encoder;
+ /*
+ * Constructs an Iterator<V> from an Iterator<Value>
+ *
+ * @param iter The source iterator
+ *
+ * @param encoder The Encoder whose decode method is used to translate from Value to V
+ */
VIterator(Iterator<Value> iter, Encoder<V> encoder) {
this.source = iter;
this.encoder = encoder;
@@ -56,6 +72,9 @@ public abstract class TypedValueCombiner
}
}
+ /*
+ * An interface for translating from byte[] to V and back.
+ */
public static interface Encoder<V> {
public byte[] encode(V v);
@@ -67,5 +86,14 @@ public abstract class TypedValueCombiner
return new Value(encoder.encode(typedReduce(key, new VIterator<V>(iter, encoder))));
}
+ /*
+ * Reduces a list of V into a single V.
+ *
+ * @param key The most recent version of the Key being reduced.
+ *
+ * @param iter An iterator over the V for different versions of the key.
+ *
+ * @return The combined V.
+ */
public abstract V typedReduce(Key key, Iterator<V> iter);
}
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/AgeOffFilter.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/AgeOffFilter.java?rev=1202816&r1=1202815&r2=1202816&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/AgeOffFilter.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/AgeOffFilter.java Wed Nov 16 18:05:09 2011
@@ -23,10 +23,9 @@ import org.apache.accumulo.core.data.Key
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.Filter;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.OptionDescriber;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-public class AgeOffFilter extends Filter implements OptionDescriber {
+public class AgeOffFilter extends Filter {
private static final String TTL = "ttl";
private long threshold;
private long currentTime;
@@ -85,6 +84,7 @@ public class AgeOffFilter extends Filter
@Override
public boolean validateOptions(Map<String,String> options) {
+ super.validateOptions(options);
try {
Long.parseLong(options.get(TTL));
} catch (NumberFormatException e) {
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ColumnAgeOffFilter.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ColumnAgeOffFilter.java?rev=1202816&r1=1202815&r2=1202816&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ColumnAgeOffFilter.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ColumnAgeOffFilter.java Wed Nov 16 18:05:09 2011
@@ -24,13 +24,12 @@ import org.apache.accumulo.core.data.Key
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.Filter;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.OptionDescriber;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.conf.ColumnToClassMapping;
import org.apache.accumulo.core.iterators.conf.PerColumnIteratorConfig;
@SuppressWarnings("deprecation")
-public class ColumnAgeOffFilter extends Filter implements OptionDescriber {
+public class ColumnAgeOffFilter extends Filter {
public ColumnAgeOffFilter() {}
@@ -100,6 +99,7 @@ public class ColumnAgeOffFilter extends
@Override
public boolean validateOptions(Map<String,String> options) {
+ super.validateOptions(options);
this.ttls = new TTLSet(options);
return true;
}
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/NoVisFilter.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/NoVisFilter.java?rev=1202816&r1=1202815&r2=1202816&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/NoVisFilter.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/NoVisFilter.java Wed Nov 16 18:05:09 2011
@@ -19,11 +19,10 @@ package org.apache.accumulo.core.iterato
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.Filter;
-import org.apache.accumulo.core.iterators.OptionDescriber;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.security.ColumnVisibility;
-public class NoVisFilter extends Filter implements OptionDescriber {
+public class NoVisFilter extends Filter {
public NoVisFilter() {}
@@ -41,7 +40,7 @@ public class NoVisFilter extends Filter
public IteratorOptions describeOptions() {
IteratorOptions io = super.describeOptions();
io.setName("novis");
- io.setDescription("NoLabelFilter hides entries without a visibility label");
+ io.setDescription("NoVisFilter hides entries without a visibility label");
return io;
}
}
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java?rev=1202816&r1=1202815&r2=1202816&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java Wed Nov 16 18:05:09 2011
@@ -26,11 +26,10 @@ import org.apache.accumulo.core.data.Key
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.Filter;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.OptionDescriber;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.util.ByteArrayBackedCharSequence;
-public class RegExFilter extends Filter implements OptionDescriber {
+public class RegExFilter extends Filter {
@Override
public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
@@ -141,6 +140,7 @@ public class RegExFilter extends Filter
@Override
public boolean validateOptions(Map<String,String> options) {
+ super.validateOptions(options);
if (options.containsKey(ROW_REGEX))
Pattern.compile(options.get(ROW_REGEX)).matcher("");
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java?rev=1202816&r1=1202815&r2=1202816&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java Wed Nov 16 18:05:09 2011
@@ -30,9 +30,9 @@ import org.apache.accumulo.core.data.Key
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.LongCombiner;
+import org.apache.accumulo.core.iterators.LongCombiner.Type;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.TypedValueCombiner;
-import org.apache.accumulo.core.iterators.LongCombiner.Type;
import org.apache.hadoop.io.WritableUtils;
public class SummingArrayCombiner extends TypedValueCombiner<List<Long>> {
@@ -90,6 +90,7 @@ public class SummingArrayCombiner extend
@Override
public boolean validateOptions(Map<String,String> options) {
+ super.validateOptions(options);
if (options.get(LongCombiner.TYPE) == null)
return false;
try {
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/TimestampFilter.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/TimestampFilter.java?rev=1202816&r1=1202815&r2=1202816&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/TimestampFilter.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/TimestampFilter.java Wed Nov 16 18:05:09 2011
@@ -24,10 +24,9 @@ import org.apache.accumulo.core.data.Key
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.Filter;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.OptionDescriber;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-public class TimestampFilter extends Filter implements OptionDescriber {
+public class TimestampFilter extends Filter {
private final SimpleDateFormat dateParser = new SimpleDateFormat("yyyyMMddHHmmssz");
public static final String START = "start";
@@ -101,6 +100,7 @@ public class TimestampFilter extends Fil
@Override
public boolean validateOptions(Map<String,String> options) {
+ super.validateOptions(options);
try {
dateParser.parse(options.get(START));
dateParser.parse(options.get(END));
Modified: incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/CombinerTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/CombinerTest.java?rev=1202816&r1=1202815&r2=1202816&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/CombinerTest.java (original)
+++ incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/CombinerTest.java Wed Nov 16 18:05:09 2011
@@ -32,10 +32,6 @@ import org.apache.accumulo.core.data.Byt
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.Combiner;
-import org.apache.accumulo.core.iterators.LongCombiner;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.accumulo.core.iterators.SortedMapIterator;
import org.apache.accumulo.core.iterators.Combiner.ValueIterator;
import org.apache.accumulo.core.iterators.TypedValueCombiner.Encoder;
import org.apache.accumulo.core.iterators.system.MultiIterator;
@@ -97,7 +93,6 @@ public class CombinerTest {
Map<String,String> opts = new HashMap<String,String>();
- opts.put(Combiner.COLUMN_PREFIX + "cf002", null);
opts.put(SummingCombiner.TYPE, SummingCombiner.Type.VARNUM.name());
ai.init(new SortedMapIterator(tm1), opts, null);