You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by bi...@apache.org on 2012/01/04 17:00:52 UTC
svn commit: r1227202 - in /incubator/accumulo/branches/1.4/src/core/src:
main/java/org/apache/accumulo/core/iterators/
main/java/org/apache/accumulo/core/iterators/user/
test/java/org/apache/accumulo/core/iterators/user/
Author: billie
Date: Wed Jan 4 16:00:51 2012
New Revision: 1227202
URL: http://svn.apache.org/viewvc?rev=1227202&view=rev
Log:
ACCUMULO-245 created setEncoder and testEncoder methods, added tests
Modified:
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java?rev=1227202&r1=1227201&r2=1227202&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java Wed Jan 4 16:00:51 2012
@@ -26,7 +26,6 @@ import java.util.Map;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
import org.apache.hadoop.io.WritableUtils;
/**
@@ -62,31 +61,18 @@ public abstract class LongCombiner exten
if (type == null)
throw new IllegalArgumentException("no type specified");
if (type.startsWith(CLASS_PREFIX)) {
- try {
- @SuppressWarnings("unchecked")
- Class<? extends Encoder<Long>> clazz = (Class<? extends Encoder<Long>>) AccumuloClassLoader.loadClass(type.substring(CLASS_PREFIX.length()),
- Encoder.class);
- encoder = clazz.newInstance();
- if (encoder.decode(encoder.encode(42l)) != 42l) {
- throw new IllegalArgumentException("something wrong with " + type + " -- doesn't encode and decode a Long properly");
- }
- } catch (ClassNotFoundException e) {
- throw new IllegalArgumentException(e);
- } catch (InstantiationException e) {
- throw new IllegalArgumentException(e);
- } catch (IllegalAccessException e) {
- throw new IllegalArgumentException(e);
- }
+ setEncoder(type.substring(CLASS_PREFIX.length()));
+ testEncoder(42l);
} else {
switch (Type.valueOf(type)) {
case VARLEN:
- encoder = VAR_LEN_ENCODER;
+ setEncoder(VAR_LEN_ENCODER);
return;
case FIXEDLEN:
- encoder = FIXED_LEN_ENCODER;
+ setEncoder(FIXED_LEN_ENCODER);
return;
case STRING:
- encoder = STRING_ENCODER;
+ setEncoder(STRING_ENCODER);
return;
default:
throw new IllegalArgumentException();
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java?rev=1227202&r1=1227201&r2=1227202&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java Wed Jan 4 16:00:51 2012
@@ -21,6 +21,7 @@ import java.util.NoSuchElementException;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
/**
* A Combiner that decodes each Value to type V before reducing, then encodes the result of typedReduce back to Value.
@@ -32,7 +33,7 @@ import org.apache.accumulo.core.data.Val
* Subclasses may implement a switch on the "type" variable to choose an Encoder in their init method.
*/
public abstract class TypedValueCombiner<V> extends Combiner {
- protected Encoder<V> encoder;
+ private Encoder<V> encoder = null;
/**
* A Java Iterator that translates an Iterator<Value> to an Iterator<V> using the decode method of an Encoder.
@@ -82,6 +83,68 @@ public abstract class TypedValueCombiner
public V decode(byte[] b);
}
+ /**
+ * Sets the Encoder<V> used to translate Values to V and back.
+ *
+ * @param encoder
+ */
+ protected void setEncoder(Encoder<V> encoder) {
+ this.encoder = encoder;
+ }
+
+ /**
+ * Instantiates and sets the Encoder<V> used to translate Values to V and back.
+ *
+ * @param encoderClass
+ * @throws IllegalArgumentException
+ * if ClassNotFoundException, InstantiationException, or IllegalAccessException occurs
+ */
+ protected void setEncoder(String encoderClass) {
+ try {
+ @SuppressWarnings("unchecked")
+ Class<? extends Encoder<V>> clazz = (Class<? extends Encoder<V>>) AccumuloClassLoader.loadClass(encoderClass, Encoder.class);
+ encoder = clazz.newInstance();
+ } catch (ClassNotFoundException e) {
+ throw new IllegalArgumentException(e);
+ } catch (InstantiationException e) {
+ throw new IllegalArgumentException(e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ /**
+ * Tests whether v remains the same when encoded and decoded with the current encoder.
+ *
+ * @param v
+ * @throws IllegalStateException
+ * if an encoder has not been set.
+ * @throws IllegalArgumentException
+ * if the test fails.
+ */
+ protected void testEncoder(V v) {
+ if (encoder == null)
+ throw new IllegalStateException("encoder has not been initialized");
+ testEncoder(encoder, v);
+ }
+
+ /**
+ * Tests whether v remains the same when encoded and decoded with the given encoder.
+ *
+ * @param encoder
+ * @param v
+ * @throws IllegalArgumentException
+ * if the test fails.
+ */
+ public static <V> void testEncoder(Encoder<V> encoder, V v) {
+ try {
+ if (!v.equals(encoder.decode(encoder.encode(v))))
+ throw new IllegalArgumentException("something wrong with " + encoder.getClass().getName() + " -- doesn't encode and decode " + v + " properly");
+ } catch (ClassCastException e) {
+ throw new IllegalArgumentException(encoder.getClass().getName() + " doesn't encode " + v.getClass().getName());
+ }
+ }
+
@Override
public Value reduce(Key key, Iterator<Value> iter) {
return new Value(encoder.encode(typedReduce(key, new VIterator<V>(iter, encoder))));
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java?rev=1227202&r1=1227201&r2=1227202&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java Wed Jan 4 16:00:51 2012
@@ -34,7 +34,6 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.iterators.LongCombiner;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.TypedValueCombiner;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
import org.apache.hadoop.io.WritableUtils;
/**
@@ -86,32 +85,18 @@ public class SummingArrayCombiner extend
if (type == null)
throw new IllegalArgumentException("no type specified");
if (type.startsWith(CLASS_PREFIX)) {
- try {
- @SuppressWarnings("unchecked")
- Class<? extends Encoder<List<Long>>> clazz = (Class<? extends Encoder<List<Long>>>) AccumuloClassLoader.loadClass(
- type.substring(CLASS_PREFIX.length()), Encoder.class);
- encoder = clazz.newInstance();
- List<Long> testList = encoder.decode(encoder.encode(Arrays.asList(0l, 1l)));
- if (testList.size() != 2 || testList.get(0) != 0l || testList.get(1) != 1l) {
- throw new IllegalArgumentException("something wrong with " + type + " -- doesn't encode and decode a List<Long> properly");
- }
- } catch (ClassNotFoundException e) {
- throw new IllegalArgumentException(e);
- } catch (InstantiationException e) {
- throw new IllegalArgumentException(e);
- } catch (IllegalAccessException e) {
- throw new IllegalArgumentException(e);
- }
+ setEncoder(type.substring(CLASS_PREFIX.length()));
+ testEncoder(Arrays.asList(0l, 1l));
} else {
switch (Type.valueOf(options.get(TYPE))) {
case VARLEN:
- encoder = new VarLongArrayEncoder();
+ setEncoder(VAR_LONG_ARRAY_ENCODER);
return;
case FIXEDLEN:
- encoder = new FixedLongArrayEncoder();
+ setEncoder(FIXED_LONG_ARRAY_ENCODER);
return;
case STRING:
- encoder = new StringArrayEncoder();
+ setEncoder(STRING_ARRAY_ENCODER);
return;
default:
throw new IllegalArgumentException();
Modified: incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java?rev=1227202&r1=1227201&r2=1227202&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java Wed Jan 4 16:00:51 2012
@@ -42,12 +42,14 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.iterators.LongCombiner.VarLenEncoder;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.SortedMapIterator;
+import org.apache.accumulo.core.iterators.TypedValueCombiner;
import org.apache.accumulo.core.iterators.TypedValueCombiner.Encoder;
import org.apache.accumulo.core.iterators.aggregation.LongSummation;
import org.apache.accumulo.core.iterators.aggregation.NumArraySummation;
import org.apache.accumulo.core.iterators.aggregation.NumSummation;
import org.apache.accumulo.core.iterators.system.MultiIterator;
import org.apache.hadoop.io.Text;
+import org.junit.Assert;
import org.junit.Test;
@SuppressWarnings("deprecation")
@@ -587,6 +589,37 @@ public class CombinerTest {
ai.next();
assertFalse(ai.hasTop());
+
+ is.clearOptions();
+ SummingArrayCombiner.setEncodingType(is, SummingCombiner.VAR_LEN_ENCODER.getClass().getName());
+ Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
+
+ try {
+ ai.init(new SortedMapIterator(tm1), is.getProperties(), null);
+ Assert.fail();
+ } catch (IllegalArgumentException e) {}
+
+ is.clearOptions();
+ SummingArrayCombiner.setEncodingType(is, BadEncoder.class.getName());
+ Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
+
+ try {
+ ai.init(new SortedMapIterator(tm1), is.getProperties(), null);
+ Assert.fail();
+ } catch (IllegalArgumentException e) {}
+ }
+
+ public static class BadEncoder implements Encoder<List<Long>> {
+ @Override
+ public byte[] encode(List<Long> v) {
+ return new byte[0];
+ }
+
+ @Override
+ public List<Long> decode(byte[] b) {
+ return new ArrayList<Long>();
+ }
+
}
@Test
@@ -596,6 +629,37 @@ public class CombinerTest {
sumArray(SummingArrayCombiner.StringArrayEncoder.class, SummingArrayCombiner.Type.STRING);
}
+ @Test
+ public void testEncoders() {
+ TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, Long.MAX_VALUE);
+ TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, Long.MIN_VALUE);
+ TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, 42l);
+ TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, -42l);
+ TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, 0l);
+ TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, Long.MAX_VALUE);
+ TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, Long.MIN_VALUE);
+ TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, 42l);
+ TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, -42l);
+ TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, 0l);
+ TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, Long.MAX_VALUE);
+ TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, Long.MIN_VALUE);
+ TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, 42l);
+ TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, -42l);
+ TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, 0l);
+
+ TypedValueCombiner.testEncoder(SummingArrayCombiner.FIXED_LONG_ARRAY_ENCODER, Arrays.asList(0l, -1l, 10l, Long.MAX_VALUE, Long.MIN_VALUE));
+ TypedValueCombiner.testEncoder(SummingArrayCombiner.VAR_LONG_ARRAY_ENCODER, Arrays.asList(0l, -1l, 10l, Long.MAX_VALUE, Long.MIN_VALUE));
+ TypedValueCombiner.testEncoder(SummingArrayCombiner.STRING_ARRAY_ENCODER, Arrays.asList(0l, -1l, 10l, Long.MAX_VALUE, Long.MIN_VALUE));
+ }
+
+ @Test
+ public void testAdds() {
+ assertEquals(LongCombiner.safeAdd(Long.MIN_VALUE + 5, -10), Long.MIN_VALUE);
+ assertEquals(LongCombiner.safeAdd(Long.MAX_VALUE - 5, 10), Long.MAX_VALUE);
+ assertEquals(LongCombiner.safeAdd(Long.MIN_VALUE + 5, -5), Long.MIN_VALUE);
+ assertEquals(LongCombiner.safeAdd(Long.MAX_VALUE - 5, 5), Long.MAX_VALUE);
+ }
+
/**
* @throws IOException
* @deprecated since 1.4