You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@accumulo.apache.org by GitBox <gi...@apache.org> on 2018/12/10 21:26:24 UTC
[GitHub] milleruntime closed pull request #812: Make new MR API accept
booleans. Closes #803
milleruntime closed pull request #812: Make new MR API accept booleans. Closes #803
URL: https://github.com/apache/accumulo/pull/812
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormat.java
index 4f89bf058f..2bffe353bc 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormat.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormat.java
@@ -43,9 +43,9 @@
* <pre>
* AccumuloInputFormat.configure().clientProperties(props).table(name) // required
* .auths(auths).addIterator(iter1).ranges(ranges).fetchColumns(columns).executionHints(hints)
- * .samplerConfiguration(sampleConf).disableAutoAdjustRanges() // enabled by default
- * .scanIsolation() // not available with batchScan()
- * .offlineScan() // not available with batchScan()
+ * .samplerConfiguration(sampleConf).autoAdjustRanges(false) // enabled by default
+ * .scanIsolation(true) // not available with batchScan()
+ * .offlineScan(true) // not available with batchScan()
* .store(job);
* </pre>
*
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloRowInputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloRowInputFormat.java
index d89e8492ad..f4ddeb3b56 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloRowInputFormat.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloRowInputFormat.java
@@ -45,9 +45,9 @@
* <pre>
* AccumuloRowInputFormat.configure().clientProperties(props).table(name) // required
* .auths(auths).addIterator(iter1).ranges(ranges).fetchColumns(columns).executionHints(hints)
- * .samplerConfiguration(sampleConf).disableAutoAdjustRanges() // enabled by default
- * .scanIsolation() // not available with batchScan()
- * .offlineScan() // not available with batchScan()
+ * .samplerConfiguration(sampleConf).autoAdjustRanges(false) // enabled by default
+ * .scanIsolation(true) // not available with batchScan()
+ * .offlineScan(true) // not available with batchScan()
* .store(job);
* </pre>
*
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java
index 383bc9219c..4677bda2f2 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java
@@ -70,40 +70,6 @@
InputFormatOptions<T> table(String tableName);
}
- /**
- * Options for batch scan
- *
- * @since 2.0
- */
- interface BatchScanOptions<T> {
- /**
- * Finish configuring, verify and store options into the JobConf or Job
- */
- void store(T t) throws AccumuloException, AccumuloSecurityException;
- }
-
- /**
- * Options for scan
- *
- * @since 2.0
- */
- interface ScanOptions<T> extends BatchScanOptions<T> {
- /**
- * @see InputFormatOptions#scanIsolation()
- */
- ScanOptions<T> scanIsolation();
-
- /**
- * @see InputFormatOptions#localIterators()
- */
- ScanOptions<T> localIterators();
-
- /**
- * @see InputFormatOptions#offlineScan()
- */
- ScanOptions<T> offlineScan();
- }
-
/**
* Optional values to set using fluent API
*
@@ -184,14 +150,14 @@
*
* @see #ranges(Collection)
*/
- InputFormatOptions<T> disableAutoAdjustRanges();
+ InputFormatOptions<T> autoAdjustRanges(boolean value);
/**
* Enables the use of the {@link IsolatedScanner} in this job.
* <p>
* By default, this feature is <b>disabled</b>.
*/
- ScanOptions<T> scanIsolation();
+ InputFormatOptions<T> scanIsolation(boolean value);
/**
* Enables the use of the {@link ClientSideIteratorScanner} in this job. This feature will cause
@@ -201,7 +167,7 @@
* <p>
* By default, this feature is <b>disabled</b>.
*/
- ScanOptions<T> localIterators();
+ InputFormatOptions<T> localIterators(boolean value);
/**
* Enable reading offline tables. By default, this feature is disabled and only online tables
@@ -228,7 +194,7 @@
* <p>
* By default, this feature is <b>disabled</b>.
*/
- ScanOptions<T> offlineScan();
+ InputFormatOptions<T> offlineScan(boolean value);
/**
* Enables the use of the {@link org.apache.accumulo.core.client.BatchScanner} in this job.
@@ -239,18 +205,18 @@
* In order to achieve good locality of InputSplits this option always clips the input Ranges to
* tablet boundaries. This may result in one input Range contributing to several InputSplits.
* <p>
- * Note: calls to {@link #disableAutoAdjustRanges()} is ignored when BatchScan is enabled.
+ * Note: calls to {@link #autoAdjustRanges(boolean)} is ignored when BatchScan is enabled.
* <p>
* This configuration is incompatible with:
* <ul>
- * <li>{@link #offlineScan()}</li>
- * <li>{@link #localIterators()}</li>
- * <li>{@link #scanIsolation()}</li>
+ * <li>{@link #offlineScan(boolean)}</li>
+ * <li>{@link #localIterators(boolean)}</li>
+ * <li>{@link #scanIsolation(boolean)}</li>
* </ul>
* <p>
* By default, this feature is <b>disabled</b>.
*/
- BatchScanOptions<T> batchScan();
+ InputFormatOptions<T> batchScan(boolean value);
/**
* Finish configuring, verify and serialize options into the JobConf or Job
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/OutputFormatBuilder.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/OutputFormatBuilder.java
index 44a5cc0f62..678ba31efc 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/OutputFormatBuilder.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/OutputFormatBuilder.java
@@ -65,7 +65,7 @@
* <p>
* By default, this feature is <b>disabled</b>.
*/
- OutputOptions<T> createTables();
+ OutputOptions<T> createTables(boolean value);
/**
* Enables the directive to use simulation mode for this job. In simulation mode, no output is
@@ -73,7 +73,7 @@
* <p>
* By default, this feature is <b>disabled</b>.
*/
- OutputOptions<T> simulationMode();
+ OutputOptions<T> simulationMode(boolean value);
/**
* Finish configuring, verify and serialize options into the Job or JobConf
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/InputFormatBuilderImpl.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/InputFormatBuilderImpl.java
index 71101ff5ab..befb467b28 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/InputFormatBuilderImpl.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/InputFormatBuilderImpl.java
@@ -43,8 +43,7 @@
public class InputFormatBuilderImpl<T>
implements InputFormatBuilder, InputFormatBuilder.ClientParams<T>,
- InputFormatBuilder.TableParams<T>, InputFormatBuilder.InputFormatOptions<T>,
- InputFormatBuilder.ScanOptions<T>, InputFormatBuilder.BatchScanOptions<T> {
+ InputFormatBuilder.TableParams<T>, InputFormatBuilder.InputFormatOptions<T> {
Class<?> callingClass;
String tableName;
@@ -134,33 +133,34 @@ public InputFormatBuilderImpl(Class<?> callingClass) {
}
@Override
- public InputFormatOptions<T> disableAutoAdjustRanges() {
- bools.autoAdjustRanges = false;
+ public InputFormatOptions<T> autoAdjustRanges(boolean value) {
+ bools.autoAdjustRanges = value;
return this;
}
@Override
- public ScanOptions<T> scanIsolation() {
- bools.scanIsolation = true;
+ public InputFormatOptions<T> scanIsolation(boolean value) {
+ bools.scanIsolation = value;
return this;
}
@Override
- public ScanOptions<T> localIterators() {
- bools.localIters = true;
+ public InputFormatOptions<T> localIterators(boolean value) {
+ bools.localIters = value;
return this;
}
@Override
- public ScanOptions<T> offlineScan() {
- bools.offlineScan = true;
+ public InputFormatOptions<T> offlineScan(boolean value) {
+ bools.offlineScan = value;
return this;
}
@Override
- public BatchScanOptions<T> batchScan() {
- bools.batchScan = true;
- bools.autoAdjustRanges = true;
+ public InputFormatOptions<T> batchScan(boolean value) {
+ bools.batchScan = value;
+ if (value)
+ bools.autoAdjustRanges = true;
return this;
}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/OutputFormatBuilderImpl.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/OutputFormatBuilderImpl.java
index defe67f348..46a15479d3 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/OutputFormatBuilderImpl.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/OutputFormatBuilderImpl.java
@@ -53,14 +53,14 @@
}
@Override
- public OutputFormatBuilder.OutputOptions<T> createTables() {
- this.createTables = true;
+ public OutputFormatBuilder.OutputOptions<T> createTables(boolean value) {
+ this.createTables = value;
return this;
}
@Override
- public OutputFormatBuilder.OutputOptions<T> simulationMode() {
- this.simulationMode = true;
+ public OutputFormatBuilder.OutputOptions<T> simulationMode(boolean value) {
+ this.simulationMode = value;
return this;
}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java
index 3d7eeab025..92d5fb7583 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java
@@ -45,7 +45,7 @@ public void setAccumuloConfigs(Job job) throws AccumuloException, AccumuloSecuri
AccumuloInputFormat.configure().clientProperties(clientProps).table(tableName).auths(auths)
.store(job);
AccumuloOutputFormat.configure().clientProperties(clientProps).defaultTable(tableName)
- .createTables().store(job);
+ .createTables(true).store(job);
}
}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java
index a2bb14ffc4..18ff93942e 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java
@@ -38,7 +38,7 @@ public void setAccumuloConfigs(Job job) throws AccumuloException, AccumuloSecuri
AccumuloInputFormat.configure().clientProperties(clientProps).table(tableName).auths(auths)
.store(job);
AccumuloOutputFormat.configure().clientProperties(clientProps).defaultTable(tableName)
- .createTables().store(job);
+ .createTables(true).store(job);
}
public String getTableName() {
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapred/AccumuloInputFormatIT.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapred/AccumuloInputFormatIT.java
index 4c8ed2dff4..ce55953dbc 100644
--- a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapred/AccumuloInputFormatIT.java
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapred/AccumuloInputFormatIT.java
@@ -125,12 +125,11 @@ public int run(String[] args) throws Exception {
InputFormatBuilder.InputFormatOptions<JobConf> opts = AccumuloInputFormat.configure()
.clientProperties(getClientInfo().getProperties()).table(table)
.auths(Authorizations.EMPTY);
- if (batchScan)
- opts.batchScan();
+
if (sample) {
opts.samplerConfiguration(SAMPLER_CONFIG);
}
- opts.store(job);
+ opts.batchScan(batchScan).store(job);
job.setMapperClass(TestMapper.class);
job.setMapOutputKeyClass(Key.class);
@@ -220,7 +219,8 @@ public void testCorrectRangeInputSplits() throws Exception {
accumuloClient.tableOperations().create(table);
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(auths).fetchColumns(fetchColumns).scanIsolation().localIterators().store(job);
+ .auths(auths).fetchColumns(fetchColumns).scanIsolation(true).localIterators(true)
+ .store(job);
AccumuloInputFormat aif = new AccumuloInputFormat();
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapreduce/AccumuloInputFormatIT.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapreduce/AccumuloInputFormatIT.java
index 90d2571a01..b0d8cef04c 100644
--- a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapreduce/AccumuloInputFormatIT.java
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapreduce/AccumuloInputFormatIT.java
@@ -20,7 +20,6 @@
import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
@@ -64,7 +63,9 @@
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
@@ -93,6 +94,9 @@ public void before() {
inputFormat = new AccumuloInputFormat();
}
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
/**
* Tests several different paths through the getSplits() method by setting different properties
* and verifying the results.
@@ -106,7 +110,7 @@ public void testGetSplits() throws Exception {
Job job = Job.getInstance();
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).scanIsolation().store(job);
+ .auths(Authorizations.EMPTY).scanIsolation(true).store(job);
// split table
TreeSet<Text> splitsToAdd = new TreeSet<>();
@@ -132,11 +136,10 @@ public void testGetSplits() throws Exception {
// offline mode
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).offlineScan().store(job);
- try {
- inputFormat.getSplits(job);
- fail("An exception should have been thrown");
- } catch (IOException e) {}
+ .auths(Authorizations.EMPTY).offlineScan(true).store(job);
+
+ exception.expect(IOException.class);
+ inputFormat.getSplits(job);
client.tableOperations().offline(table, true);
splits = inputFormat.getSplits(job);
@@ -148,22 +151,21 @@ public void testGetSplits() throws Exception {
// overlapping ranges
ranges.add(new Range(String.format("%09d", i), String.format("%09d", i + 2)));
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).ranges(ranges).offlineScan().store(job);
+ .auths(Authorizations.EMPTY).ranges(ranges).offlineScan(true).store(job);
splits = inputFormat.getSplits(job);
assertEquals(2, splits.size());
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).disableAutoAdjustRanges().offlineScan().store(job);
+ .auths(Authorizations.EMPTY).autoAdjustRanges(true).offlineScan(true).store(job);
splits = inputFormat.getSplits(job);
assertEquals(ranges.size(), splits.size());
// BatchScan not available for offline scans
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).batchScan().store(job);
- try {
- inputFormat.getSplits(job);
- fail("An exception should have been thrown");
- } catch (IOException e) {}
+ .auths(Authorizations.EMPTY).batchScan(true).store(job);
+
+ exception.expect(IllegalArgumentException.class);
+ inputFormat.getSplits(job);
// table online tests
client.tableOperations().online(table, true);
@@ -175,20 +177,20 @@ public void testGetSplits() throws Exception {
// BatchScan not available with isolated iterators
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).scanIsolation().store(job);
+ .auths(Authorizations.EMPTY).scanIsolation(true).store(job);
splits = inputFormat.getSplits(job);
assertEquals(2, splits.size());
// BatchScan not available with local iterators
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).localIterators().store(job);
+ .auths(Authorizations.EMPTY).localIterators(true).store(job);
splits = inputFormat.getSplits(job);
assertEquals(2, splits.size());
AccumuloInputFormat.configure().clientProperties(getClientInfo().getProperties()).table(table)
- .auths(Authorizations.EMPTY).batchScan().store(job);
+ .auths(Authorizations.EMPTY).batchScan(true).store(job);
// Check we are getting back correct type pf split
splits = inputFormat.getSplits(job);
@@ -289,10 +291,8 @@ public int run(String[] args) throws Exception {
.auths(Authorizations.EMPTY);
if (sample)
opts = opts.samplerConfiguration(SAMPLER_CONFIG);
- if (batchScan)
- opts.batchScan().store(job);
- else
- opts.store(job);
+
+ opts.batchScan(batchScan).store(job);
job.setMapperClass(TestMapper.class);
job.setMapOutputKeyClass(Key.class);
@@ -401,14 +401,13 @@ public void testCorrectRangeInputSplits() throws Exception {
.singleton(new IteratorSetting.Column(new Text("foo"), new Text("bar")));
Collection<Pair<Text,Text>> fetchColumnsText = Collections
.singleton(new Pair<>(new Text("foo"), new Text("bar")));
- boolean isolated = true, localIters = true;
AccumuloClient accumuloClient = getAccumuloClient();
accumuloClient.tableOperations().create(table);
InputFormatOptions<Job> opts = AccumuloInputFormat.configure()
.clientProperties(getClientInfo().getProperties()).table(table).auths(auths);
- opts.fetchColumns(fetchColumns).scanIsolation().localIterators().store(job);
+ opts.fetchColumns(fetchColumns).scanIsolation(true).localIterators(true).store(job);
AccumuloInputFormat aif = new AccumuloInputFormat();
@@ -423,8 +422,8 @@ public void testCorrectRangeInputSplits() throws Exception {
RangeInputSplit risplit = (RangeInputSplit) split;
assertEquals(table, risplit.getTableName());
- assertEquals(isolated, risplit.isIsolatedScan());
- assertEquals(localIters, risplit.usesLocalIterators());
+ assertEquals(true, risplit.isIsolatedScan());
+ assertEquals(true, risplit.usesLocalIterators());
assertEquals(fetchColumnsText, risplit.getFetchedColumns());
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services