You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/06/26 22:23:19 UTC
[3/7] git commit: ACCUMULO-2649 Get some more tests for the mapred
RangeInputSplit
ACCUMULO-2649 Get some more tests for the mapred RangeInputSplit
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2422d499
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2422d499
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2422d499
Branch: refs/heads/master
Commit: 2422d49972fc388d12a0168e3aaa15082d9e0740
Parents: 50baaf1
Author: Josh Elser <el...@apache.org>
Authored: Thu Jun 26 15:47:03 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jun 26 15:47:03 2014 -0400
----------------------------------------------------------------------
.../client/mapred/AccumuloInputFormatTest.java | 55 ++++++++++
.../core/client/mapred/RangeInputSplitTest.java | 105 +++++++++++++++++++
.../mapreduce/AccumuloInputFormatTest.java | 2 +-
3 files changed, 161 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2422d499/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
index 3ec9bb1..402e687 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
@@ -23,11 +23,14 @@ import static org.junit.Assert.assertTrue;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
@@ -36,10 +39,13 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.user.RegExFilter;
import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.Pair;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Mapper;
@@ -48,6 +54,8 @@ import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Level;
+import org.junit.Assert;
import org.junit.Test;
public class AccumuloInputFormatTest {
@@ -277,4 +285,51 @@ public class AccumuloInputFormatTest {
assertNull(e1);
assertNull(e2);
}
+
+ @Test
+ public void testCorrectRangeInputSplits() throws Exception {
+ JobConf job = new JobConf();
+
+ String username = "user", table = "table", instance = "mapred_testCorrectRangeInputSplits";
+ PasswordToken password = new PasswordToken("password");
+ Authorizations auths = new Authorizations("foo");
+ Collection<Pair<Text,Text>> fetchColumns = Collections.singleton(new Pair<Text,Text>(new Text("foo"), new Text("bar")));
+ boolean isolated = true, localIters = true;
+ Level level = Level.WARN;
+
+ Instance inst = new MockInstance(instance);
+ Connector connector = inst.getConnector(username, password);
+ connector.tableOperations().create(table);
+
+ AccumuloInputFormat.setConnectorInfo(job, username, password);
+ AccumuloInputFormat.setInputTableName(job, table);
+ AccumuloInputFormat.setScanAuthorizations(job, auths);
+ AccumuloInputFormat.setMockInstance(job, instance);
+ AccumuloInputFormat.setScanIsolation(job, isolated);
+ AccumuloInputFormat.setLocalIterators(job, localIters);
+ AccumuloInputFormat.fetchColumns(job, fetchColumns);
+ AccumuloInputFormat.setLogLevel(job, level);
+
+ AccumuloInputFormat aif = new AccumuloInputFormat();
+
+ InputSplit[] splits = aif.getSplits(job, 1);
+
+ Assert.assertEquals(1, splits.length);
+
+ InputSplit split = splits[0];
+
+ Assert.assertEquals(RangeInputSplit.class, split.getClass());
+
+ RangeInputSplit risplit = (RangeInputSplit) split;
+
+ Assert.assertEquals(username, risplit.getPrincipal());
+ Assert.assertEquals(table, risplit.getTable());
+ Assert.assertEquals(password, risplit.getToken());
+ Assert.assertEquals(auths, risplit.getAuths());
+ Assert.assertEquals(instance, risplit.getInstanceName());
+ Assert.assertEquals(isolated, risplit.isIsolatedScan());
+ Assert.assertEquals(localIters, risplit.usesLocalIterators());
+ Assert.assertEquals(fetchColumns, risplit.getFetchedColumns());
+ Assert.assertEquals(level, risplit.getLogLevel());
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2422d499/core/src/test/java/org/apache/accumulo/core/client/mapred/RangeInputSplitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapred/RangeInputSplitTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapred/RangeInputSplitTest.java
new file mode 100644
index 0000000..590f2b5
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/RangeInputSplitTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class RangeInputSplitTest {
+
+ @Test
+ public void testSimpleWritable() throws IOException {
+ RangeInputSplit split = new RangeInputSplit(new Range(new Key("a"), new Key("b")), new String[]{"localhost"});
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ split.write(dos);
+
+ RangeInputSplit newSplit = new RangeInputSplit();
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dis = new DataInputStream(bais);
+ newSplit.readFields(dis);
+
+ Assert.assertEquals(split.getRange(), newSplit.getRange());
+ Assert.assertTrue(Arrays.equals(split.getLocations(), newSplit.getLocations()));
+ }
+
+ @Test
+ public void testAllFieldsWritable() throws IOException {
+ RangeInputSplit split = new RangeInputSplit(new Range(new Key("a"), new Key("b")), new String[]{"localhost"});
+
+ Set<Pair<Text,Text>> fetchedColumns = new HashSet<Pair<Text,Text>>();
+
+ fetchedColumns.add(new Pair<Text,Text>(new Text("colf1"), new Text("colq1")));
+ fetchedColumns.add(new Pair<Text,Text>(new Text("colf2"), new Text("colq2")));
+
+ split.setAuths(new Authorizations("foo"));
+ split.setOffline(true);
+ split.setIsolatedScan(true);
+ split.setUsesLocalIterators(true);
+ split.setFetchedColumns(fetchedColumns);
+ split.setToken(new PasswordToken("password"));
+ split.setPrincipal("root");
+ split.setInstanceName("instance");
+ split.setMockInstance(true);
+ split.setZooKeepers("localhost");
+ split.setLogLevel(Level.WARN);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ split.write(dos);
+
+ RangeInputSplit newSplit = new RangeInputSplit();
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dis = new DataInputStream(bais);
+ newSplit.readFields(dis);
+
+ Assert.assertEquals(split.getRange(), newSplit.getRange());
+ Assert.assertArrayEquals(split.getLocations(), newSplit.getLocations());
+
+ Assert.assertEquals(split.getAuths(), newSplit.getAuths());
+ Assert.assertEquals(split.isOffline(), newSplit.isOffline());
+ Assert.assertEquals(split.isIsolatedScan(), newSplit.isOffline());
+ Assert.assertEquals(split.usesLocalIterators(), newSplit.usesLocalIterators());
+ Assert.assertEquals(split.getFetchedColumns(), newSplit.getFetchedColumns());
+ Assert.assertEquals(split.getToken(), newSplit.getToken());
+ Assert.assertEquals(split.getPrincipal(), newSplit.getPrincipal());
+ Assert.assertEquals(split.getInstanceName(), newSplit.getInstanceName());
+ Assert.assertEquals(split.isMockInstance(), newSplit.isMockInstance());
+ Assert.assertEquals(split.getZooKeepers(), newSplit.getZooKeepers());
+ Assert.assertEquals(split.getLogLevel(), newSplit.getLogLevel());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2422d499/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
index ae5e395..9095486 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
@@ -333,7 +333,7 @@ public class AccumuloInputFormatTest {
public void testCorrectRangeInputSplits() throws Exception {
Job job = new Job(new Configuration(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
- String username = "user", table = "table", instance = "instance";
+ String username = "user", table = "table", instance = "mapreduce_testCorrectRangeInputSplits";
PasswordToken password = new PasswordToken("password");
Authorizations auths = new Authorizations("foo");
Collection<Pair<Text,Text>> fetchColumns = Collections.singleton(new Pair<Text,Text>(new Text("foo"), new Text("bar")));