You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by bi...@apache.org on 2012/01/10 16:16:37 UTC
svn commit: r1229588 - in /incubator/accumulo/branches/1.4/src/core/src:
main/java/org/apache/accumulo/core/iterators/
main/java/org/apache/accumulo/core/iterators/user/
test/java/org/apache/accumulo/core/iterators/user/
Author: billie
Date: Tue Jan 10 15:16:37 2012
New Revision: 1229588
URL: http://svn.apache.org/viewvc?rev=1229588&view=rev
Log:
ACCUMULO-289 added all columns option to Combiner, tested and improved options describing for combiners
Modified:
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java
incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java?rev=1229588&r1=1229587&r2=1229588&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java Tue Jan 10 15:16:37 2012
@@ -19,7 +19,6 @@ package org.apache.accumulo.core.iterato
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -46,6 +45,7 @@ import org.apache.log4j.Logger;
public abstract class Combiner extends WrappingIterator implements OptionDescriber {
static final Logger log = Logger.getLogger(Combiner.class);
protected static final String COLUMNS_OPTION = "columns";
+ protected static final String ALL_OPTION = "all";
/**
* A Java Iterator that iterates over the Values for a given Key from a source SortedKeyValueIterator.
@@ -156,7 +156,7 @@ public abstract class Combiner extends W
// check if aggregation is needed
if (super.hasTop()) {
workKey.set(super.getTopKey());
- if (combiners.contains(workKey)) {
+ if (combineAllColumns || combiners.contains(workKey)) {
if (workKey.isDeleted())
return;
topKey = workKey;
@@ -205,11 +205,18 @@ public abstract class Combiner extends W
public abstract Value reduce(Key key, Iterator<Value> iter);
private ColumnSet combiners;
+ private boolean combineAllColumns;
@Override
public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
super.init(source, options, env);
+ combineAllColumns = false;
+ if (options.containsKey(ALL_OPTION)) {
+ combineAllColumns = Boolean.parseBoolean(options.get(ALL_OPTION));
+ if (combineAllColumns)
+ return;
+ }
if (!options.containsKey(COLUMNS_OPTION))
throw new IllegalArgumentException("Must specify " + COLUMNS_OPTION + " option");
@@ -222,30 +229,32 @@ public abstract class Combiner extends W
@Override
public IteratorOptions describeOptions() {
- return new IteratorOptions(
- "comb",
- "Combiners apply reduce functions to values with identical keys",
- Collections
- .singletonMap(
- COLUMNS_OPTION,
- "<col fam>[:<col qual>]{,<col fam>[:<col qual>]} escape non-alphanum chars using %<hex>."),
- null);
+ IteratorOptions io = new IteratorOptions("comb", "Combiners apply reduce functions to values with identical keys", null, null);
+ io.addNamedOption(ALL_OPTION, "set to true to apply Combiner to every column, otherwise leave blank. if true, " + COLUMNS_OPTION
+ + " option will be ignored.");
+ io.addNamedOption(COLUMNS_OPTION, "<col fam>[:<col qual>]{,<col fam>[:<col qual>]} escape non-alphanum chars using %<hex>.");
+ return io;
}
@Override
public boolean validateOptions(Map<String,String> options) {
+ if (options.containsKey(ALL_OPTION)) {
+ combineAllColumns = Boolean.parseBoolean(options.get(ALL_OPTION));
+ if (combineAllColumns)
+ return true;
+ }
if (!options.containsKey(COLUMNS_OPTION))
return false;
String encodedColumns = options.get(COLUMNS_OPTION);
if (encodedColumns.length() == 0)
return false;
-
+
for (String columns : encodedColumns.split(",")) {
if (!ColumnSet.isValidEncoding(columns))
return false;
}
-
+
return true;
}
@@ -270,4 +279,17 @@ public abstract class Combiner extends W
is.addOption(COLUMNS_OPTION, sb.toString());
}
+
+ /**
+ * A convenience method to set the "all columns" option on a Combiner. If true, the columns option will be ignored and the Combiner will be applied to all
+ * columns.
+ *
+ * @param is
+ * iterator settings object to configure
+ * @param enableAllColumns
+ * if true the combiner will be applied to all columns
+ */
+ public static void setCombineAllColumns(IteratorSetting is, boolean combineAllColumns) {
+ is.addOption(ALL_OPTION, Boolean.toString(combineAllColumns));
+ }
}
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java?rev=1229588&r1=1229587&r2=1229588&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java Tue Jan 10 15:16:37 2012
@@ -35,4 +35,12 @@ public class MaxCombiner extends LongCom
}
return max;
}
+
+ @Override
+ public IteratorOptions describeOptions() {
+ IteratorOptions io = super.describeOptions();
+ io.setName("max");
+ io.setDescription("MaxCombiner interprets Values as Longs and finds their maximum. A variety of encodings (variable length, fixed length, or string) are available");
+ return io;
+ }
}
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java?rev=1229588&r1=1229587&r2=1229588&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java Tue Jan 10 15:16:37 2012
@@ -35,4 +35,12 @@ public class MinCombiner extends LongCom
}
return min;
}
+
+ @Override
+ public IteratorOptions describeOptions() {
+ IteratorOptions io = super.describeOptions();
+ io.setName("min");
+ io.setDescription("MinCombiner interprets Values as Longs and finds their minimum. A variety of encodings (variable length, fixed length, or string) are available");
+ return io;
+ }
}
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java?rev=1229588&r1=1229587&r2=1229588&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java Tue Jan 10 15:16:37 2012
@@ -107,8 +107,8 @@ public class SummingArrayCombiner extend
@Override
public IteratorOptions describeOptions() {
IteratorOptions io = super.describeOptions();
- io.setName("summingarraycombiner");
- io.setDescription("SummingArrayCombiner can interpret Values as arrays of Longs using a variety of encodings (arrays of variable length longs or fixed length longs, or comma-separated strings) before combining");
+ io.setName("sumarray");
+ io.setDescription("SummingArrayCombiner can interpret Values as arrays of Longs using a variety of encodings (arrays of variable length longs or fixed length longs, or comma-separated strings) before summing element-wise.");
io.addNamedOption(TYPE, "<VARLEN|FIXEDLEN|STRING|fullClassName>");
return io;
}
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java?rev=1229588&r1=1229587&r2=1229588&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java Tue Jan 10 15:16:37 2012
@@ -33,4 +33,12 @@ public class SummingCombiner extends Lon
}
return sum;
}
+
+ @Override
+ public IteratorOptions describeOptions() {
+ IteratorOptions io = super.describeOptions();
+ io.setName("sum");
+ io.setDescription("SummingCombiner interprets Values as Longs and adds them together. A variety of encodings (variable length, fixed length, or string) are available");
+ return io;
+ }
}
Modified: incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java?rev=1229588&r1=1229587&r2=1229588&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java Tue Jan 10 15:16:37 2012
@@ -351,6 +351,34 @@ public class CombinerTest {
assertEquals(nk(2, 2, 1, 2), ai.getTopKey());
assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
+ // combine all columns
+
+ is = new IteratorSetting(1, SummingCombiner.class);
+ LongCombiner.setEncodingType(is, SummingCombiner.Type.STRING);
+ Combiner.setCombineAllColumns(is, true);
+
+ ai.init(new SortedMapIterator(tm1), is.getProperties(), null);
+ ai.seek(new Range(), EMPTY_COL_FAMS, false);
+
+ assertTrue(ai.hasTop());
+ assertEquals(nk(0, 0, 1, 1), ai.getTopKey());
+ assertEquals("7", encoder.decode(ai.getTopValue().get()).toString());
+
+ ai.next();
+
+ assertTrue(ai.hasTop());
+ assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
+ assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
+
+ ai.next();
+
+ assertTrue(ai.hasTop());
+ assertEquals(nk(2, 2, 1, 2), ai.getTopKey());
+ assertEquals("5", encoder.decode(ai.getTopValue().get()).toString());
+
+ ai.next();
+
+ assertFalse(ai.hasTop());
}
@Test