You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2013/11/24 01:33:58 UTC
[27/31] git commit: ACCUMULO-1854 Update tests for hadoop-2
compatibility
ACCUMULO-1854 Update tests for hadoop-2 compatibility
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c88d87aa
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c88d87aa
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c88d87aa
Branch: refs/heads/master
Commit: c88d87aa08819ed94fa9aae8583985b4fcf8e588
Parents: fae8282
Author: Josh Elser <el...@apache.org>
Authored: Sat Nov 23 13:49:14 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Sat Nov 23 13:49:14 2013 -0500
----------------------------------------------------------------------
.../mapreduce/AccumuloInputFormatTest.java | 198 ++++++++-----------
.../BadPasswordSplitsAccumuloInputFormat.java | 26 +++
.../EmptySplitsAccumuloInputFormat.java | 21 ++
3 files changed, 129 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c88d87aa/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
index f9ccdf1..93dba65 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@@ -46,8 +47,10 @@ import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -60,11 +63,9 @@ import org.junit.Assert;
import org.junit.Test;
public class AccumuloInputFormatTest {
-
+
private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
- private static final String INSTANCE_NAME = PREFIX + "_mapreduce_instance";
- private static final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1";
-
+
/**
* Test basic setting & getting of max versions.
*
@@ -79,7 +80,7 @@ public class AccumuloInputFormatTest {
int version = AccumuloInputFormat.getMaxVersions(job.getConfiguration());
assertEquals(1, version);
}
-
+
/**
* Test max versions with an invalid value.
*
@@ -92,7 +93,7 @@ public class AccumuloInputFormatTest {
Job job = new Job();
AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 0);
}
-
+
/**
* Test no max version configured.
*
@@ -104,7 +105,7 @@ public class AccumuloInputFormatTest {
Job job = new Job();
assertEquals(-1, AccumuloInputFormat.getMaxVersions(job.getConfiguration()));
}
-
+
/**
* Check that the iterator configuration is getting stored in the Job conf correctly.
*
@@ -113,7 +114,7 @@ public class AccumuloInputFormatTest {
@Test
public void testSetIterator() throws IOException {
Job job = new Job();
-
+
IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
AccumuloInputFormat.addIterator(job, is);
Configuration conf = job.getConfiguration();
@@ -122,36 +123,36 @@ public class AccumuloInputFormatTest {
String iterators = conf.get("AccumuloInputFormat.ScanOpts.Iterators");
assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
}
-
+
@Test
public void testAddIterator() throws IOException {
Job job = new Job();
-
+
AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
iter.addOption("v1", "1");
iter.addOption("junk", "\0omg:!\\xyzzy");
AccumuloInputFormat.addIterator(job, iter);
-
+
List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
-
+
// Check the list size
assertTrue(list.size() == 3);
-
+
// Walk the list and make sure our settings are correct
IteratorSetting setting = list.get(0);
assertEquals(1, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass());
assertEquals("WholeRow", setting.getName());
assertEquals(0, setting.getOptions().size());
-
+
setting = list.get(1);
assertEquals(2, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
assertEquals("Versions", setting.getName());
assertEquals(0, setting.getOptions().size());
-
+
setting = list.get(2);
assertEquals(3, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
@@ -160,7 +161,7 @@ public class AccumuloInputFormatTest {
assertEquals("1", setting.getOptions().get("v1"));
assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
}
-
+
/**
* Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR character (':') and ITERATOR_SEPARATOR (',') characters. There
* should be no exceptions thrown when trying to parse these types of option entries.
@@ -175,12 +176,12 @@ public class AccumuloInputFormatTest {
someSetting.addOption(key, value);
Job job = new Job();
AccumuloInputFormat.addIterator(job, someSetting);
-
+
List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
assertEquals(1, list.size());
assertEquals(1, list.get(0).getOptions().size());
assertEquals(list.get(0).getOptions().get(key), value);
-
+
someSetting.addOption(key + "2", value);
someSetting.setPriority(2);
someSetting.setName("it2");
@@ -193,7 +194,7 @@ public class AccumuloInputFormatTest {
assertEquals(list.get(1).getOptions().get(key), value);
assertEquals(list.get(1).getOptions().get(key + "2"), value);
}
-
+
/**
* Test getting iterator settings for multiple iterators set
*
@@ -202,55 +203,55 @@ public class AccumuloInputFormatTest {
@Test
public void testGetIteratorSettings() throws IOException {
Job job = new Job();
-
+
AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
-
+
List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
-
+
// Check the list size
assertTrue(list.size() == 3);
-
+
// Walk the list and make sure our settings are correct
IteratorSetting setting = list.get(0);
assertEquals(1, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
assertEquals("WholeRow", setting.getName());
-
+
setting = list.get(1);
assertEquals(2, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
assertEquals("Versions", setting.getName());
-
+
setting = list.get(2);
assertEquals(3, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
assertEquals("Count", setting.getName());
-
+
}
-
+
@Test
public void testSetRegex() throws IOException {
Job job = new Job();
-
+
String regex = ">\"*%<>\'\\";
-
+
IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
RegExFilter.setRegexs(is, regex, null, null, null, false);
AccumuloInputFormat.addIterator(job, is);
-
+
assertTrue(regex.equals(AccumuloInputFormat.getIterators(job).get(0).getName()));
}
-
+
private static AssertionError e1 = null;
private static AssertionError e2 = null;
-
+
private static class MRTester extends Configured implements Tool {
private static class TestMapper extends Mapper<Key,Value,Key,Value> {
Key key = null;
int count = 0;
-
+
@Override
protected void map(Key k, Value v, Context context) throws IOException, InterruptedException {
try {
@@ -264,7 +265,7 @@ public class AccumuloInputFormatTest {
key = new Key(k);
count++;
}
-
+
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
try {
@@ -274,46 +275,52 @@ public class AccumuloInputFormatTest {
}
}
}
-
+
@Override
public int run(String[] args) throws Exception {
-
- if (args.length != 3) {
- throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table>");
+
+ if (args.length != 5) {
+ throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table> <instanceName> <inputFormatClass>");
}
-
+
String user = args[0];
String pass = args[1];
String table = args[2];
-
+ String instanceName = args[3];
+ String inputFormatClassName = args[4];
+ Class<? extends InputFormat> inputFormatClass = (Class<? extends InputFormat>) Class.forName(inputFormatClassName);
+
Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
job.setJarByClass(this.getClass());
-
- job.setInputFormatClass(AccumuloInputFormat.class);
-
+
+ job.setInputFormatClass(inputFormatClass);
+
AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(pass));
AccumuloInputFormat.setInputTableName(job, table);
- AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
-
+ AccumuloInputFormat.setMockInstance(job, instanceName);
+
job.setMapperClass(TestMapper.class);
job.setMapOutputKeyClass(Key.class);
job.setMapOutputValueClass(Value.class);
job.setOutputFormatClass(NullOutputFormat.class);
-
+
job.setNumReduceTasks(0);
-
+
job.waitForCompletion(true);
-
+
return job.isSuccessful() ? 0 : 1;
}
-
- public static void main(String[] args) throws Exception {
- assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
+
+ public static int main(String[] args) throws Exception {
+ return ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args);
}
}
-
+
@Test
public void testMap() throws Exception {
+ final String INSTANCE_NAME = PREFIX + "_mapreduce_instance";
+ final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1";
+
MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
Connector c = mockInstance.getConnector("root", new PasswordToken(""));
c.tableOperations().create(TEST_TABLE_1);
@@ -324,13 +331,12 @@ public class AccumuloInputFormatTest {
bw.addMutation(m);
}
bw.close();
-
- MRTester.main(new String[] {"root", "", TEST_TABLE_1});
+
+ Assert.assertEquals(0, MRTester.main(new String[] {"root", "", TEST_TABLE_1, INSTANCE_NAME, AccumuloInputFormat.class.getCanonicalName()}));
assertNull(e1);
assertNull(e2);
}
-
- @SuppressWarnings("deprecation")
+
@Test
public void testCorrectRangeInputSplits() throws Exception {
Job job = new Job(new Configuration(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
@@ -354,19 +360,19 @@ public class AccumuloInputFormatTest {
AccumuloInputFormat.setLocalIterators(job, localIters);
AccumuloInputFormat.fetchColumns(job, fetchColumns);
AccumuloInputFormat.setLogLevel(job, level);
-
+
AccumuloInputFormat aif = new AccumuloInputFormat();
-
+
List<InputSplit> splits = aif.getSplits(job);
-
+
Assert.assertEquals(1, splits.size());
-
+
InputSplit split = splits.get(0);
-
+
Assert.assertEquals(RangeInputSplit.class, split.getClass());
-
+
RangeInputSplit risplit = (RangeInputSplit) split;
-
+
Assert.assertEquals(username, risplit.getPrincipal());
Assert.assertEquals(table, risplit.getTable());
Assert.assertEquals(password, risplit.getToken());
@@ -377,7 +383,7 @@ public class AccumuloInputFormatTest {
Assert.assertEquals(fetchColumns, risplit.getFetchedColumns());
Assert.assertEquals(level, risplit.getLogLevel());
}
-
+
static class TestMapper extends Mapper<Key,Value,Key,Value> {
Key key = null;
int count = 0;
@@ -397,7 +403,7 @@ public class AccumuloInputFormatTest {
public void testPartialInputSplitDelegationToConfiguration() throws Exception {
String user = "testPartialInputSplitUser";
PasswordToken password = new PasswordToken("");
-
+
MockInstance mockInstance = new MockInstance("testPartialInputSplitDelegationToConfiguration");
Connector c = mockInstance.getConnector(user, password);
c.tableOperations().create("testtable");
@@ -409,37 +415,17 @@ public class AccumuloInputFormatTest {
}
bw.close();
- Job job = new Job(new Configuration());
- job.setInputFormatClass(AccumuloInputFormat.class);
- job.setMapperClass(TestMapper.class);
- job.setNumReduceTasks(0);
- AccumuloInputFormat.setConnectorInfo(job, user, password);
- AccumuloInputFormat.setInputTableName(job, "testtable");
- AccumuloInputFormat.setScanAuthorizations(job, new Authorizations());
- AccumuloInputFormat.setMockInstance(job, "testPartialInputSplitDelegationToConfiguration");
-
- AccumuloInputFormat input = new AccumuloInputFormat();
- List<InputSplit> splits = input.getSplits(job);
- assertEquals(splits.size(), 1);
-
- TestMapper mapper = (TestMapper) job.getMapperClass().newInstance();
-
- RangeInputSplit emptySplit = new RangeInputSplit();
-
- // Using an empty split should fall back to the information in the Job's Configuration
- TaskAttemptID id = new TaskAttemptID();
- TaskAttemptContext attempt = new TaskAttemptContext(job.getConfiguration(), id);
- RecordReader<Key,Value> reader = input.createRecordReader(emptySplit, attempt);
- Mapper<Key,Value,Key,Value>.Context context = mapper.new Context(job.getConfiguration(), id, reader, null, null, null, emptySplit);
- reader.initialize(emptySplit, context);
- mapper.run(context);
+ Assert.assertEquals(0, MRTester.main(new String[] {user, "", "testtable", "testPartialInputSplitDelegationToConfiguration",
+ EmptySplitsAccumuloInputFormat.class.getCanonicalName()}));
+ assertNull(e1);
+ assertNull(e2);
}
- @Test(expected = IOException.class)
+ @Test
public void testPartialFailedInputSplitDelegationToConfiguration() throws Exception {
String user = "testPartialFailedInputSplit";
PasswordToken password = new PasswordToken("");
-
+
MockInstance mockInstance = new MockInstance("testPartialFailedInputSplitDelegationToConfiguration");
Connector c = mockInstance.getConnector(user, password);
c.tableOperations().create("testtable");
@@ -451,30 +437,10 @@ public class AccumuloInputFormatTest {
}
bw.close();
- Job job = new Job(new Configuration());
- job.setInputFormatClass(AccumuloInputFormat.class);
- job.setMapperClass(TestMapper.class);
- job.setNumReduceTasks(0);
- AccumuloInputFormat.setConnectorInfo(job, user, password);
- AccumuloInputFormat.setInputTableName(job, "testtable");
- AccumuloInputFormat.setMockInstance(job, "testPartialFailedInputSplitDelegationToConfiguration");
-
- AccumuloInputFormat input = new AccumuloInputFormat();
- List<InputSplit> splits = input.getSplits(job);
- assertEquals(splits.size(), 1);
-
- TestMapper mapper = (TestMapper) job.getMapperClass().newInstance();
-
- RangeInputSplit emptySplit = new RangeInputSplit();
- emptySplit.setPrincipal("root");
- emptySplit.setToken(new PasswordToken("anythingelse"));
-
- // Using an empty split should fall back to the information in the Job's Configuration
- TaskAttemptID id = new TaskAttemptID();
- TaskAttemptContext attempt = new TaskAttemptContext(job.getConfiguration(), id);
- RecordReader<Key,Value> reader = input.createRecordReader(emptySplit, attempt);
- Mapper<Key,Value,Key,Value>.Context context = mapper.new Context(job.getConfiguration(), id, reader, null, null, null, emptySplit);
- reader.initialize(emptySplit, context);
- mapper.run(context);
+ // We should fail before we even get into the Mapper because we can't make the RecordReader
+ Assert.assertEquals(1, MRTester.main(new String[] {user, "", "testtable", "testPartialFailedInputSplitDelegationToConfiguration",
+ BadPasswordSplitsAccumuloInputFormat.class.getCanonicalName()}));
+ assertNull(e1);
+ assertNull(e2);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c88d87aa/core/src/test/java/org/apache/accumulo/core/client/mapreduce/BadPasswordSplitsAccumuloInputFormat.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/BadPasswordSplitsAccumuloInputFormat.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/BadPasswordSplitsAccumuloInputFormat.java
new file mode 100644
index 0000000..846208b
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/BadPasswordSplitsAccumuloInputFormat.java
@@ -0,0 +1,26 @@
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+
+/**
+ * AccumuloInputFormat which returns an "empty" RangeInputSplit
+ */
+public class BadPasswordSplitsAccumuloInputFormat extends AccumuloInputFormat {
+
+ @Override
+ public List<InputSplit> getSplits(JobContext context) throws IOException {
+ List<InputSplit> splits = super.getSplits(context);
+
+ for (InputSplit split : splits) {
+ RangeInputSplit rangeSplit = (RangeInputSplit) split;
+ rangeSplit.setToken(new PasswordToken("anythingelse"));
+ }
+
+ return splits;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c88d87aa/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java
new file mode 100644
index 0000000..91db378
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java
@@ -0,0 +1,21 @@
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+
+/**
+ * AccumuloInputFormat which returns an "empty" RangeInputSplit
+ */
+public class EmptySplitsAccumuloInputFormat extends AccumuloInputFormat {
+
+ @Override
+ public List<InputSplit> getSplits(JobContext context) throws IOException {
+ super.getSplits(context);
+
+ return Arrays.<InputSplit> asList(new RangeInputSplit());
+ }
+}
\ No newline at end of file