You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2015/06/19 22:46:44 UTC
accumulo git commit: ACCUMULO-3423 tightended some timings,
added more m/r options required to run the tests,
figured out how to get distcp to work when already running in m/r,
used Assume methods to bail on tests that fail in the m/r environment
Repository: accumulo
Updated Branches:
refs/heads/master 979acacc9 -> b7f88f9df
ACCUMULO-3423 tightended some timings, added more m/r options required to run the tests, figured out how to get distcp to work when already running in m/r, used Assume methods to bail on tests that fail in the m/r environment
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b7f88f9d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b7f88f9d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b7f88f9d
Branch: refs/heads/master
Commit: b7f88f9dfb3d244d7b1b684b4c225514b868e9e0
Parents: 979acac
Author: Eric C. Newton <er...@gmail.com>
Authored: Fri Jun 19 16:46:37 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Fri Jun 19 16:46:37 2015 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/test/DumpConfigIT.java | 8 +++----
.../org/apache/accumulo/test/ShellServerIT.java | 8 +++----
.../accumulo/test/functional/ExamplesIT.java | 18 ++++++----------
.../test/functional/MetadataMaxFilesIT.java | 17 +++++++--------
.../accumulo/test/functional/ReadWriteIT.java | 4 ++--
.../test/mrit/IntegrationTestMapReduce.java | 22 ++++++++++++--------
.../accumulo/test/start/KeywordStartIT.java | 6 ++----
7 files changed, 39 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7f88f9d/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java b/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java
index 5cc37a5..01d873e 100644
--- a/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java
@@ -31,15 +31,11 @@ import org.apache.accumulo.server.util.Admin;
import org.apache.accumulo.test.functional.ConfigurableMacBase;
import org.apache.accumulo.test.functional.FunctionalTestUtils;
import org.apache.hadoop.conf.Configuration;
-import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
public class DumpConfigIT extends ConfigurableMacBase {
- @Rule
- public TemporaryFolder folder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
-
@Override
public int defaultTimeoutSeconds() {
return 2 * 60;
@@ -52,6 +48,10 @@ public class DumpConfigIT extends ConfigurableMacBase {
@Test
public void test() throws Exception {
+ File target = new File(System.getProperty("user.dir"), "target");
+ target.mkdirs();
+ TemporaryFolder folder = new TemporaryFolder(target);
+ folder.create();
File siteFileBackup = new File(folder.getRoot(), "accumulo-site.xml.bak");
assertFalse(siteFileBackup.exists());
assertEquals(0, exec(Admin.class, new String[] {"dumpConfig", "-a", "-d", folder.getRoot().getPath()}).waitFor());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7f88f9d/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
index e01de42..cf22df5 100644
--- a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
@@ -315,7 +315,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
String exportUri = "file://" + exportDir.toString();
String localTmp = "file://" + new File(rootPath, "ShellServerIT.tmp").toString();
ts.exec("exporttable -t " + table + " " + exportUri, true);
- DistCp cp = newDistCp();
+ DistCp cp = newDistCp(new Configuration(false));
String import_ = "file://" + new File(rootPath, "ShellServerIT.import").toString();
if (getCluster().getClientConfig().getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
// DistCp bugs out trying to get a fs delegation token to perform the cp. Just copy it ourselves by hand.
@@ -355,7 +355,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
ts.exec("deletetable -f " + table2, true);
}
- private DistCp newDistCp() {
+ private DistCp newDistCp(Configuration conf) {
try {
@SuppressWarnings("unchecked")
Constructor<DistCp>[] constructors = (Constructor<DistCp>[]) DistCp.class.getConstructors();
@@ -363,9 +363,9 @@ public class ShellServerIT extends SharedMiniClusterBase {
Class<?>[] parameterTypes = constructor.getParameterTypes();
if (parameterTypes.length > 0 && parameterTypes[0].equals(Configuration.class)) {
if (parameterTypes.length == 1) {
- return constructor.newInstance(new Configuration());
+ return constructor.newInstance(conf);
} else if (parameterTypes.length == 2) {
- return constructor.newInstance(new Configuration(), null);
+ return constructor.newInstance(conf, null);
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7f88f9d/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
index b43cda5..671a6bb 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
@@ -20,6 +20,7 @@ import static com.google.common.base.Charsets.UTF_8;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.io.IOException;
@@ -95,7 +96,6 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.util.Tool;
import org.junit.After;
import org.junit.Assert;
-import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@@ -235,10 +235,7 @@ public class ExamplesIT extends AccumuloClusterHarness {
default:
throw new RuntimeException("Unknown cluster type");
}
- if (!new File(dirListDirectory).exists()) {
- log.info("Skipping test: there's no input here");
- return;
- }
+ assumeTrue(new File(dirListDirectory).exists());
// Index a directory listing on /tmp. If this is running against a standalone cluster, we can't guarantee Accumulo source will be there.
if (saslEnabled) {
args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "--dirTable", dirTable, "--indexTable", indexTable, "--dataTable",
@@ -383,10 +380,7 @@ public class ExamplesIT extends AccumuloClusterHarness {
@Test
public void testShardedIndex() throws Exception {
File src = new File(System.getProperty("user.dir") + "/src");
- if (!src.exists()) {
- log.info("Skipping test: src does not exist");
- return;
- }
+ assumeTrue(src.exists());
String[] names = getUniqueNames(3);
final String shard = names[0], index = names[1];
c.tableOperations().create(shard);
@@ -449,7 +443,7 @@ public class ExamplesIT extends AccumuloClusterHarness {
@Test
public void testBulkIngest() throws Exception {
// TODO Figure out a way to run M/R with Kerberos
- Assume.assumeTrue(getAdminToken() instanceof PasswordToken);
+ assumeTrue(getAdminToken() instanceof PasswordToken);
String tableName = getUniqueNames(1)[0];
FileSystem fs = getFileSystem();
Path p = new Path(dir, "tmp");
@@ -482,7 +476,7 @@ public class ExamplesIT extends AccumuloClusterHarness {
@Test
public void testTeraSortAndRead() throws Exception {
// TODO Figure out a way to run M/R with Kerberos
- Assume.assumeTrue(getAdminToken() instanceof PasswordToken);
+ assumeTrue(getAdminToken() instanceof PasswordToken);
String tableName = getUniqueNames(1)[0];
String[] args;
if (saslEnabled) {
@@ -525,7 +519,7 @@ public class ExamplesIT extends AccumuloClusterHarness {
@Test
public void testWordCount() throws Exception {
// TODO Figure out a way to run M/R with Kerberos
- Assume.assumeTrue(getAdminToken() instanceof PasswordToken);
+ assumeTrue(getAdminToken() instanceof PasswordToken);
String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
is = new IteratorSetting(10, SummingCombiner.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7f88f9d/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
index 086dd1a..977a659 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
@@ -18,8 +18,6 @@ package org.apache.accumulo.test.functional;
import static org.junit.Assert.assertEquals;
-import java.util.HashMap;
-import java.util.Map;
import java.util.Map.Entry;
import java.util.SortedSet;
import java.util.TreeSet;
@@ -49,10 +47,9 @@ public class MetadataMaxFilesIT extends ConfigurableMacBase {
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- Map<String,String> siteConfig = new HashMap<String,String>();
- siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
- siteConfig.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "10");
- cfg.setSiteConfig(siteConfig);
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "1");
+ cfg.setProperty(Property.TSERV_SCAN_MAX_OPENFILES, "10");
+ cfg.setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT, "100");
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@@ -69,6 +66,8 @@ public class MetadataMaxFilesIT extends ConfigurableMacBase {
splits.add(new Text(String.format("%03d", i)));
}
c.tableOperations().setProperty(MetadataTable.NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(), "10000");
+ // propagation time
+ UtilWaitThread.sleep(5 * 1000);
for (int i = 0; i < 5; i++) {
String tableName = "table" + i;
log.info("Creating " + tableName);
@@ -79,22 +78,21 @@ public class MetadataMaxFilesIT extends ConfigurableMacBase {
c.tableOperations().flush(MetadataTable.NAME, null, null, true);
c.tableOperations().flush(RootTable.NAME, null, null, true);
}
- UtilWaitThread.sleep(20 * 1000);
log.info("shutting down");
assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
cluster.stop();
log.info("starting up");
cluster.start();
- UtilWaitThread.sleep(30 * 1000);
+ Credentials creds = new Credentials("root", new PasswordToken(ROOT_PASSWORD));
while (true) {
MasterMonitorInfo stats = null;
- Credentials creds = new Credentials("root", new PasswordToken(ROOT_PASSWORD));
Client client = null;
try {
ClientContext context = new ClientContext(c.getInstance(), creds, getClientConfig());
client = MasterClient.getConnectionWithRetry(context);
+ log.info("Fetching stats");
stats = client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
} finally {
if (client != null)
@@ -108,6 +106,7 @@ public class MetadataMaxFilesIT extends ConfigurableMacBase {
tablets += entry.getValue().onlineTablets;
}
}
+ log.info("Online tablets " + tablets);
if (tablets == 5005)
break;
UtilWaitThread.sleep(1000);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7f88f9d/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
index d048071..485d6d2 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
@@ -107,7 +107,7 @@ public class ReadWriteIT extends AccumuloClusterHarness {
private static final Logger log = LoggerFactory.getLogger(ReadWriteIT.class);
- static final int ROWS = 200000;
+ static final int ROWS = 100000;
static final int COLS = 1;
static final String COLF = "colf";
@@ -443,7 +443,7 @@ public class ReadWriteIT extends AccumuloClusterHarness {
@Test
public void localityGroupChange() throws Exception {
- // Make changes to locality groups and ensure nothing is lostssh
+ // Make changes to locality groups and ensure nothing is lost
final Connector connector = getConnector();
String table = getUniqueNames(1)[0];
TableOperations to = connector.tableOperations();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7f88f9d/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java b/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java
index 9cb807c..2573f8e 100644
--- a/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java
+++ b/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java
@@ -24,7 +24,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
@@ -63,11 +62,9 @@ import org.slf4j.LoggerFactory;
* </pre>
*
* Run the class below as a map-reduce job, giving it the lists of tests, and a place to store the results.
+ *
* <pre>
- * $ yarn jar lib/accumulo-test-mrit.jar \
- * -libjars lib/native/libaccumulo.so \
- * -Dmapreduce.map.memory.mb=4000 \
- * /tmp/tests /tmp/results
+ * $ yarn jar lib/accumulo-test-mrit.jar -libjars lib/native/libaccumulo.so /tmp/tests /tmp/results
* </pre>
*
* The result is a list of IT classes that pass or fail. Those classes that fail will be annotated with the particular test that failed within the class.
@@ -162,16 +159,23 @@ public class IntegrationTestMapReduce extends Configured implements Tool {
}
Configuration conf = getConf();
Job job = Job.getInstance(conf, "accumulo integration test runner");
+ conf = job.getConfiguration();
- // no need to run a test multiple times
- job.setSpeculativeExecution(false);
+ // some tests take more than 10 minutes
+ conf.setLong(MRJobConfig.TASK_TIMEOUT, 20 * 60 * 1000);
+
+ // minicluster uses a lot of ram
+ conf.setInt(MRJobConfig.MAP_MEMORY_MB, 4000);
// hadoop puts an ancient version of jline on the classpath
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
+ // no need to run a test multiple times
+ job.setSpeculativeExecution(false);
+
// read one line at a time
job.setInputFormatClass(NLineInputFormat.class);
- conf.setInt(NLineInputFormat.LINES_PER_MAP, 1);
+ NLineInputFormat.setNumLinesPerSplit(job, 1);
// run the test
job.setJarByClass(IntegrationTestMapReduce.class);
@@ -179,7 +183,7 @@ public class IntegrationTestMapReduce extends Configured implements Tool {
// group test by result code
job.setReducerClass(TestReducer.class);
- job.setOutputKeyClass(IntWritable.class);
+ job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7f88f9d/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
index 9201bea..40f6822 100644
--- a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.test.start;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.io.IOException;
@@ -93,10 +94,7 @@ public class KeywordStartIT {
// Note: this test may fail in Eclipse, if the services files haven't been generated by the AutoService annotation processor
@Test
public void testExpectedClasses() throws IOException {
- if (!new File(System.getProperty("user.dir") + "/src").exists()) {
- log.info("Ignoring test: probably running under map-reduce");
- return;
- }
+ assumeTrue(new File(System.getProperty("user.dir") + "/src").exists());
TreeMap<String,Class<? extends KeywordExecutable>> expectSet = new TreeMap<>();
expectSet.put("admin", Admin.class);
expectSet.put("classpath", Classpath.class);