You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by vi...@apache.org on 2012/01/23 20:46:20 UTC
svn commit: r1234945 - in /incubator/accumulo/trunk: ./ src/core/
src/core/src/main/java/org/apache/accumulo/core/iterators/
src/core/src/main/java/org/apache/accumulo/core/iterators/user/
Author: vines
Date: Mon Jan 23 19:46:20 2012
New Revision: 1234945
URL: http://svn.apache.org/viewvc?rev=1234945&view=rev
Log:
ACCUMULO-319 - merging
Added:
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
- copied unchanged from r1234942, incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
Modified:
incubator/accumulo/trunk/ (props changed)
incubator/accumulo/trunk/src/core/ (props changed)
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Jan 23 19:46:20 2012
@@ -1,3 +1,3 @@
/incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043
/incubator/accumulo/branches/1.3.5rc:1209938
-/incubator/accumulo/branches/1.4:1201902-1234099
+/incubator/accumulo/branches/1.4:1201902-1234942
Propchange: incubator/accumulo/trunk/src/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Jan 23 19:46:20 2012
@@ -1,3 +1,3 @@
/incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215
/incubator/accumulo/branches/1.3.5rc/src/core:1209938
-/incubator/accumulo/branches/1.4/src/core:1201902-1234099
+/incubator/accumulo/branches/1.4/src/core:1201902-1234942
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java?rev=1234945&r1=1234944&r2=1234945&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java Mon Jan 23 19:46:20 2012
@@ -120,7 +120,7 @@ public abstract class LongCombiner exten
try {
return WritableUtils.readVLong(dis);
} catch (IOException e) {
- throw new NumberFormatException(e.getMessage());
+ throw new ValueFormatException(e);
}
}
}
@@ -150,7 +150,7 @@ public abstract class LongCombiner exten
public static long decode(byte[] b, int offset) {
if (b.length < offset + 8)
- throw new NumberFormatException("trying to convert to long, but byte array isn't long enough, wanted " + (offset + 8) + " found " + b.length);
+ throw new ValueFormatException("trying to convert to long, but byte array isn't long enough, wanted " + (offset + 8) + " found " + b.length);
return (((long) b[offset + 0] << 56) + ((long) (b[offset + 1] & 255) << 48) + ((long) (b[offset + 2] & 255) << 40) + ((long) (b[offset + 3] & 255) << 32)
+ ((long) (b[offset + 4] & 255) << 24) + ((b[offset + 5] & 255) << 16) + ((b[offset + 6] & 255) << 8) + ((b[offset + 7] & 255) << 0));
}
@@ -167,7 +167,11 @@ public abstract class LongCombiner exten
@Override
public Long decode(byte[] b) {
- return Long.parseLong(new String(b));
+ try {
+ return Long.parseLong(new String(b));
+ } catch (NumberFormatException nfe) {
+ throw new ValueFormatException(nfe);
+ }
}
}
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java?rev=1234945&r1=1234944&r2=1234945&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java Mon Jan 23 19:46:20 2012
@@ -16,7 +16,9 @@
*/
package org.apache.accumulo.core.iterators;
+import java.io.IOException;
import java.util.Iterator;
+import java.util.Map;
import java.util.NoSuchElementException;
import org.apache.accumulo.core.data.Key;
@@ -34,13 +36,17 @@ import org.apache.accumulo.start.classlo
*/
public abstract class TypedValueCombiner<V> extends Combiner {
private Encoder<V> encoder = null;
+ private boolean lossy = false;
+ protected static final String LOSSY = "lossy";
+
/**
* A Java Iterator that translates an Iterator<Value> to an Iterator<V> using the decode method of an Encoder.
*/
private static class VIterator<V> implements Iterator<V> {
private Iterator<Value> source;
private Encoder<V> encoder;
+ private boolean lossy;
/**
* Constructs an Iterator<V> from an Iterator<Value>
@@ -50,22 +56,44 @@ public abstract class TypedValueCombiner
*
* @param encoder
* The Encoder whose decode method is used to translate from Value to V
+ *
+ * @param lossy
+ * Determines whether to error on failure to decode or ignore and move on
*/
- VIterator(Iterator<Value> iter, Encoder<V> encoder) {
+ VIterator(Iterator<Value> iter, Encoder<V> encoder, boolean lossy) {
this.source = iter;
this.encoder = encoder;
+ this.lossy = lossy;
}
+ V next = null;
@Override
public boolean hasNext() {
- return source.hasNext();
+ if (next != null)
+ return true;
+
+ while (true)
+ {
+ if (!source.hasNext())
+ return false;
+ try
+ {
+ next = encoder.decode(source.next().get());
+ return true;
+ } catch (ValueFormatException vfe) {
+ if (!lossy)
+ throw vfe;
+ }
+ }
}
@Override
public V next() {
- if (!source.hasNext())
+ if (!hasNext())
throw new NoSuchElementException();
- return encoder.decode(source.next().get());
+ V toRet = next;
+ next = null;
+ return toRet;
}
@Override
@@ -80,7 +108,7 @@ public abstract class TypedValueCombiner
public static interface Encoder<V> {
public byte[] encode(V v);
- public V decode(byte[] b);
+ public V decode(byte[] b) throws ValueFormatException;
}
/**
@@ -155,9 +183,37 @@ public abstract class TypedValueCombiner
@Override
public Value reduce(Key key, Iterator<Value> iter) {
- return new Value(encoder.encode(typedReduce(key, new VIterator<V>(iter, encoder))));
+ return new Value(encoder.encode(typedReduce(key, new VIterator<V>(iter, encoder, lossy))));
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ setLossyness(options);
+ }
+
+ private void setLossyness(Map<String,String> options) {
+ String loss = options.get(LOSSY);
+ if (loss == null)
+ lossy = false;
+ else
+ lossy = Boolean.parseBoolean(loss);
+ }
+
+ @Override
+ public IteratorOptions describeOptions() {
+ IteratorOptions io = super.describeOptions();
+ io.addNamedOption(LOSSY, "if true, failed decodes are ignored. Otherwise combiner will error on failed decodes (default false): <TRUE|FALSE>");
+ return io;
}
+ @Override
+ public boolean validateOptions(Map<String,String> options) {
+ super.validateOptions(options);
+ setLossyness(options);
+ return true;
+ }
+
/**
* Reduces a list of V into a single V.
*
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java?rev=1234945&r1=1234944&r2=1234945&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java Mon Jan 23 19:46:20 2012
@@ -34,6 +34,7 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.iterators.LongCombiner;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.TypedValueCombiner;
+import org.apache.accumulo.core.iterators.ValueFormatException;
import org.apache.hadoop.io.WritableUtils;
/**
@@ -151,7 +152,7 @@ public class SummingArrayCombiner extend
}
return vl;
} catch (IOException e) {
- throw new NumberFormatException(e.getMessage());
+ throw new ValueFormatException(e);
}
}
}
@@ -201,7 +202,11 @@ public class SummingArrayCombiner extend
if (s.length() == 0)
la.add(0l);
else
- la.add(Long.parseLong(s));
+ try {
+ la.add(Long.parseLong(s));
+ } catch (NumberFormatException nfe) {
+ throw new ValueFormatException(nfe);
+ }
}
return la;
}