You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mm...@apache.org on 2018/12/10 21:26:26 UTC
[accumulo] branch master updated: Make new MR API accept booleans.
Closes #803 (#812)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new 9859a72 Make new MR API accept booleans. Closes #803 (#812)
9859a72 is described below
commit 9859a72b157ea331a6024c67ae9694cb9669986d
Author: Mike Miller <mm...@apache.org>
AuthorDate: Mon Dec 10 16:26:22 2018 -0500
Make new MR API accept booleans. Closes #803 (#812)
---
.../hadoop/mapreduce/AccumuloInputFormat.java | 6 +--
.../hadoop/mapreduce/AccumuloRowInputFormat.java | 6 +--
.../hadoop/mapreduce/InputFormatBuilder.java | 52 ++++------------------
.../hadoop/mapreduce/OutputFormatBuilder.java | 4 +-
.../mapreduce/InputFormatBuilderImpl.java | 26 +++++------
.../mapreduce/OutputFormatBuilderImpl.java | 8 ++--
.../lib/MapReduceClientOnDefaultTable.java | 2 +-
.../lib/MapReduceClientOnRequiredTable.java | 2 +-
.../hadoop/its/mapred/AccumuloInputFormatIT.java | 8 ++--
.../its/mapreduce/AccumuloInputFormatIT.java | 49 ++++++++++----------
10 files changed, 64 insertions(+), 99 deletions(-)
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormat.java
index 4f89bf0..2bffe35 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormat.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormat.java
@@ -43,9 +43,9 @@ import org.slf4j.LoggerFactory;
* <pre>
* AccumuloInputFormat.configure().clientProperties(props).table(name) // required
* .auths(auths).addIterator(iter1).ranges(ranges).fetchColumns(columns).executionHints(hints)
- * .samplerConfiguration(sampleConf).disableAutoAdjustRanges() // enabled by default
- * .scanIsolation() // not available with batchScan()
- * .offlineScan() // not available with batchScan()
+ * .samplerConfiguration(sampleConf).autoAdjustRanges(false) // enabled by default
+ * .scanIsolation(true) // not available with batchScan()
+ * .offlineScan(true) // not available with batchScan()
* .store(job);
* </pre>
*
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloRowInputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloRowInputFormat.java
index d89e849..f4ddeb3 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloRowInputFormat.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloRowInputFormat.java
@@ -45,9 +45,9 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
* <pre>
* AccumuloRowInputFormat.configure().clientProperties(props).table(name) // required
* .auths(auths).addIterator(iter1).ranges(ranges).fetchColumns(columns).executionHints(hints)
- * .samplerConfiguration(sampleConf).disableAutoAdjustRanges() // enabled by default
- * .scanIsolation() // not available with batchScan()
- * .offlineScan() // not available with batchScan()
+ * .samplerConfiguration(sampleConf).autoAdjustRanges(false) // enabled by default
+ * .scanIsolation(true) // not available with batchScan()
+ * .offlineScan(true) // not available with batchScan()
* .store(job);
* </pre>
*
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java
index 383bc92..4677bda 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java
@@ -71,40 +71,6 @@ public interface InputFormatBuilder {
}
/**
- * Options for batch scan
- *
- * @since 2.0
- */
- interface BatchScanOptions<T> {
- /**
- * Finish configuring, verify and store options into the JobConf or Job
- */
- void store(T t) throws AccumuloException, AccumuloSecurityException;
- }
-
- /**
- * Options for scan
- *
- * @since 2.0
- */
- interface ScanOptions<T> extends BatchScanOptions<T> {
- /**
- * @see InputFormatOptions#scanIsolation()
- */
- ScanOptions<T> scanIsolation();
-
- /**
- * @see InputFormatOptions#localIterators()
- */
- ScanOptions<T> localIterators();
-
- /**
- * @see InputFormatOptions#offlineScan()
- */
- ScanOptions<T> offlineScan();
- }
-
- /**
* Optional values to set using fluent API
*
* @since 2.0
@@ -184,14 +150,14 @@ public interface InputFormatBuilder {
*
* @see #ranges(Collection)
*/
- InputFormatOptions<T> disableAutoAdjustRanges();
+ InputFormatOptions<T> autoAdjustRanges(boolean value);
/**
* Enables the use of the {@link IsolatedScanner} in this job.
* <p>
* By default, this feature is <b>disabled</b>.
*/
- ScanOptions<T> scanIsolation();
+ InputFormatOptions<T> scanIsolation(boolean value);
/**
* Enables the use of the {@link ClientSideIteratorScanner} in this job. This feature will cause
@@ -201,7 +167,7 @@ public interface InputFormatBuilder {
* <p>
* By default, this feature is <b>disabled</b>.
*/
- ScanOptions<T> localIterators();
+ InputFormatOptions<T> localIterators(boolean value);
/**
* Enable reading offline tables. By default, this feature is disabled and only online tables
@@ -228,7 +194,7 @@ public interface InputFormatBuilder {
* <p>
* By default, this feature is <b>disabled</b>.
*/
- ScanOptions<T> offlineScan();
+ InputFormatOptions<T> offlineScan(boolean value);
/**
* Enables the use of the {@link org.apache.accumulo.core.client.BatchScanner} in this job.
@@ -239,18 +205,18 @@ public interface InputFormatBuilder {
* In order to achieve good locality of InputSplits this option always clips the input Ranges to
* tablet boundaries. This may result in one input Range contributing to several InputSplits.
* <p>
- * Note: calls to {@link #disableAutoAdjustRanges()} is ignored when BatchScan is enabled.
+ * Note: calls to {@link #autoAdjustRanges(boolean)} is ignored when BatchScan is enabled.
* <p>
* This configuration is incompatible with:
* <ul>
- * <li>{@link #offlineScan()}</li>
- * <li>{@link #localIterators()}</li>
- * <li>{@link #scanIsolation()}</li>
+ * <li>{@link #offlineScan(boolean)}</li>
+ * <li>{@link #localIterators(boolean)}</li>
+ * <li>{@link #scanIsolation(boolean)}</li>
* </ul>
* <p>
* By default, this feature is <b>disabled</b>.
*/
- BatchScanOptions<T> batchScan();
+ InputFormatOptions<T> batchScan(boolean value);
/**
* Finish configuring, verify and serialize options into the JobConf or Job
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/OutputFormatBuilder.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/OutputFormatBuilder.java
index 44a5cc0..678ba31 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/OutputFormatBuilder.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/OutputFormatBuilder.java
@@ -65,7 +65,7 @@ public interface OutputFormatBuilder {
* <p>
* By default, this feature is <b>disabled</b>.
*/
- OutputOptions<T> createTables();
+ OutputOptions<T> createTables(boolean value);
/**
* Enables the directive to use simulation mode for this job. In simulation mode, no output is
@@ -73,7 +73,7 @@ public interface OutputFormatBuilder {
* <p>
* By default, this feature is <b>disabled</b>.
*/
- OutputOptions<T> simulationMode();
+ OutputOptions<T> simulationMode(boolean value);
/**
* Finish configuring, verify and serialize options into the Job or JobConf
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/InputFormatBuilderImpl.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/InputFormatBuilderImpl.java
index a2876c1..dc4bf4e 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/InputFormatBuilderImpl.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/InputFormatBuilderImpl.java
@@ -43,8 +43,7 @@ import com.google.common.collect.ImmutableMap;
public class InputFormatBuilderImpl<T>
implements InputFormatBuilder, InputFormatBuilder.ClientParams<T>,
- InputFormatBuilder.TableParams<T>, InputFormatBuilder.InputFormatOptions<T>,
- InputFormatBuilder.ScanOptions<T>, InputFormatBuilder.BatchScanOptions<T> {
+ InputFormatBuilder.TableParams<T>, InputFormatBuilder.InputFormatOptions<T> {
Class<?> callingClass;
String tableName;
@@ -134,33 +133,34 @@ public class InputFormatBuilderImpl<T>
}
@Override
- public InputFormatOptions<T> disableAutoAdjustRanges() {
- bools.autoAdjustRanges = false;
+ public InputFormatOptions<T> autoAdjustRanges(boolean value) {
+ bools.autoAdjustRanges = value;
return this;
}
@Override
- public ScanOptions<T> scanIsolation() {
- bools.scanIsolation = true;
+ public InputFormatOptions<T> scanIsolation(boolean value) {
+ bools.scanIsolation = value;
return this;
}
@Override
- public ScanOptions<T> localIterators() {
- bools.localIters = true;
+ public InputFormatOptions<T> localIterators(boolean value) {
+ bools.localIters = value;
return this;
}
@Override
- public ScanOptions<T> offlineScan() {
- bools.offlineScan = true;
+ public InputFormatOptions<T> offlineScan(boolean value) {
+ bools.offlineScan = value;
return this;
}
@Override
- public BatchScanOptions<T> batchScan() {
- bools.batchScan = true;
- bools.autoAdjustRanges = true;
+ public InputFormatOptions<T> batchScan(boolean value) {
+ bools.batchScan = value;
+ if (value)
+ bools.autoAdjustRanges = true;
return this;
}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/OutputFormatBuilderImpl.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/OutputFormatBuilderImpl.java
index e5f0539..217f01f 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/OutputFormatBuilderImpl.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/OutputFormatBuilderImpl.java
@@ -53,14 +53,14 @@ public class OutputFormatBuilderImpl<T>
}
@Override
- public OutputFormatBuilder.OutputOptions<T> createTables() {
- this.createTables = true;
+ public OutputFormatBuilder.OutputOptions<T> createTables(boolean value) {
+ this.createTables = value;
return this;
}
@Override
- public OutputFormatBuilder.OutputOptions<T> simulationMode() {
- this.simulationMode = true;
+ public OutputFormatBuilder.OutputOptions<T> simulationMode(boolean value) {
+ this.simulationMode = value;
return this;
}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java
index 3d7eeab..92d5fb7 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java
@@ -45,7 +45,7 @@ public class MapReduceClientOnDefaultTable extends MapReduceClientOpts {
AccumuloInputFormat.configure().clientProperties(clientProps).table(tableName).auths(auths)
.store(job);
AccumuloOutputFormat.configure().clientProperties(clientProps).defaultTable(tableName)
- .createTables().store(job);
+ .createTables(true).store(job);
}
}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java
index a2bb14f..18ff939 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java
@@ -38,7 +38,7 @@ public class MapReduceClientOnRequiredTable extends MapReduceClientOpts {
AccumuloInputFormat.configure().clientProperties(clientProps).table(tableName).auths(auths)
.store(job);
AccumuloOutputFormat.configure().clientProperties(clientProps).defaultTable(tableName)
- .createTables().store(job);
+ .createTables(true).store(job);
}
public String getTableName() {
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapred/AccumuloInputFormatIT.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapred/AccumuloInputFormatIT.java
index 4c8ed2d..ce55953 100644
--- a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapred/AccumuloInputFormatIT.java
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapred/AccumuloInputFormatIT.java
@@ -125,12 +125,11 @@ public class AccumuloInputFormatIT extends AccumuloClusterHarness {
InputFormatBuilder.InputFormatOptions<JobConf> opts = AccumuloInputFormat.configure()
.clientProperties(getClientInfo().getProperties()).table(table)
.auths(Authorizations.EMPTY);
- if (batchScan)
- opts.batchScan();
+
if (sample) {
opts.samplerConfiguration(SAMPLER_CONFIG);
}
- opts.store(job);
+ opts.batchScan(batchScan).store(job);
job.setMapperClass(TestMapper.class);
job.setMapOutputKeyClass(Key.class);
@@ -220,7 +219,8 @@ public class AccumuloInputFormatIT extends AccumuloClusterHarness {
accumuloClient.tableOperations().create(table);
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(auths).fetchColumns(fetchColumns).scanIsolation().localIterators().store(job);
+ .auths(auths).fetchColumns(fetchColumns).scanIsolation(true).localIterators(true)
+ .store(job);
AccumuloInputFormat aif = new AccumuloInputFormat();
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapreduce/AccumuloInputFormatIT.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapreduce/AccumuloInputFormatIT.java
index 90d2571..b0d8cef 100644
--- a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapreduce/AccumuloInputFormatIT.java
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapreduce/AccumuloInputFormatIT.java
@@ -20,7 +20,6 @@ import static java.lang.System.currentTimeMillis;
import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
@@ -64,7 +63,9 @@ import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
@@ -93,6 +94,9 @@ public class AccumuloInputFormatIT extends AccumuloClusterHarness {
inputFormat = new AccumuloInputFormat();
}
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
/**
* Tests several different paths through the getSplits() method by setting different properties
* and verifying the results.
@@ -106,7 +110,7 @@ public class AccumuloInputFormatIT extends AccumuloClusterHarness {
Job job = Job.getInstance();
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).scanIsolation().store(job);
+ .auths(Authorizations.EMPTY).scanIsolation(true).store(job);
// split table
TreeSet<Text> splitsToAdd = new TreeSet<>();
@@ -132,11 +136,10 @@ public class AccumuloInputFormatIT extends AccumuloClusterHarness {
// offline mode
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).offlineScan().store(job);
- try {
- inputFormat.getSplits(job);
- fail("An exception should have been thrown");
- } catch (IOException e) {}
+ .auths(Authorizations.EMPTY).offlineScan(true).store(job);
+
+ exception.expect(IOException.class);
+ inputFormat.getSplits(job);
client.tableOperations().offline(table, true);
splits = inputFormat.getSplits(job);
@@ -148,22 +151,21 @@ public class AccumuloInputFormatIT extends AccumuloClusterHarness {
// overlapping ranges
ranges.add(new Range(String.format("%09d", i), String.format("%09d", i + 2)));
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).ranges(ranges).offlineScan().store(job);
+ .auths(Authorizations.EMPTY).ranges(ranges).offlineScan(true).store(job);
splits = inputFormat.getSplits(job);
assertEquals(2, splits.size());
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).disableAutoAdjustRanges().offlineScan().store(job);
+ .auths(Authorizations.EMPTY).autoAdjustRanges(true).offlineScan(true).store(job);
splits = inputFormat.getSplits(job);
assertEquals(ranges.size(), splits.size());
// BatchScan not available for offline scans
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).batchScan().store(job);
- try {
- inputFormat.getSplits(job);
- fail("An exception should have been thrown");
- } catch (IOException e) {}
+ .auths(Authorizations.EMPTY).batchScan(true).store(job);
+
+ exception.expect(IllegalArgumentException.class);
+ inputFormat.getSplits(job);
// table online tests
client.tableOperations().online(table, true);
@@ -175,20 +177,20 @@ public class AccumuloInputFormatIT extends AccumuloClusterHarness {
// BatchScan not available with isolated iterators
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).scanIsolation().store(job);
+ .auths(Authorizations.EMPTY).scanIsolation(true).store(job);
splits = inputFormat.getSplits(job);
assertEquals(2, splits.size());
// BatchScan not available with local iterators
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).localIterators().store(job);
+ .auths(Authorizations.EMPTY).localIterators(true).store(job);
splits = inputFormat.getSplits(job);
assertEquals(2, splits.size());
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).batchScan().store(job);
+ .auths(Authorizations.EMPTY).batchScan(true).store(job);
// Check we are getting back correct type pf split
splits = inputFormat.getSplits(job);
@@ -289,10 +291,8 @@ public class AccumuloInputFormatIT extends AccumuloClusterHarness {
.auths(Authorizations.EMPTY);
if (sample)
opts = opts.samplerConfiguration(SAMPLER_CONFIG);
- if (batchScan)
- opts.batchScan().store(job);
- else
- opts.store(job);
+
+ opts.batchScan(batchScan).store(job);
job.setMapperClass(TestMapper.class);
job.setMapOutputKeyClass(Key.class);
@@ -401,14 +401,13 @@ public class AccumuloInputFormatIT extends AccumuloClusterHarness {
.singleton(new IteratorSetting.Column(new Text("foo"), new Text("bar")));
Collection<Pair<Text,Text>> fetchColumnsText = Collections
.singleton(new Pair<>(new Text("foo"), new Text("bar")));
- boolean isolated = true, localIters = true;
AccumuloClient accumuloClient = getAccumuloClient();
accumuloClient.tableOperations().create(table);
InputFormatOptions<Job> opts = AccumuloInputFormat.configure()
.clientProperties(getClientInfo().getProperties()).table(table).auths(auths);
- opts.fetchColumns(fetchColumns).scanIsolation().localIterators().store(job);
+ opts.fetchColumns(fetchColumns).scanIsolation(true).localIterators(true).store(job);
AccumuloInputFormat aif = new AccumuloInputFormat();
@@ -423,8 +422,8 @@ public class AccumuloInputFormatIT extends AccumuloClusterHarness {
RangeInputSplit risplit = (RangeInputSplit) split;
assertEquals(table, risplit.getTableName());
- assertEquals(isolated, risplit.isIsolatedScan());
- assertEquals(localIters, risplit.usesLocalIterators());
+ assertEquals(true, risplit.isIsolatedScan());
+ assertEquals(true, risplit.usesLocalIterators());
assertEquals(fetchColumnsText, risplit.getFetchedColumns());
}