You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mw...@apache.org on 2017/02/01 21:12:21 UTC
accumulo-testing git commit: ACCUMULO-4579 Fixed hadoop config bug in
accumulo-testing
Repository: accumulo-testing
Updated Branches:
refs/heads/master bef8ed208 -> 28ca75a36
ACCUMULO-4579 Fixed hadoop config bug in accumulo-testing
* Bugs were causing tests to not work on cluster
* TestEnv now returns Hadoop configuration that is loaded with configuration
retrieved from accumulo-testing.properties
* Fixed bug where Twill was being improperly configured to look in wrong
location for shaded jar when running test application in YARN.
* Updated accumulo-testing.properites by adding accumulo to several property names
Project: http://git-wip-us.apache.org/repos/asf/accumulo-testing/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-testing/commit/28ca75a3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-testing/tree/28ca75a3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-testing/diff/28ca75a3
Branch: refs/heads/master
Commit: 28ca75a368168a035f1936fcca8d3a4a520e6eb3
Parents: bef8ed2
Author: Mike Walch <mw...@apache.org>
Authored: Mon Jan 30 15:03:21 2017 -0500
Committer: Mike Walch <mw...@apache.org>
Committed: Wed Feb 1 16:07:35 2017 -0500
----------------------------------------------------------------------
conf/accumulo-testing.properties.example | 28 ++++++++------
.../apache/accumulo/testing/core/TestEnv.java | 39 +++++++++++++-------
.../apache/accumulo/testing/core/TestProps.java | 25 ++++++++-----
.../core/continuous/ContinuousBatchWalker.java | 2 +-
.../testing/yarn/YarnAccumuloTestRunner.java | 19 ++++++----
5 files changed, 68 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/28ca75a3/conf/accumulo-testing.properties.example
----------------------------------------------------------------------
diff --git a/conf/accumulo-testing.properties.example b/conf/accumulo-testing.properties.example
index 1d43e12..ce868be 100644
--- a/conf/accumulo-testing.properties.example
+++ b/conf/accumulo-testing.properties.example
@@ -23,24 +23,28 @@ test.common.accumulo.instance=instance
test.common.accumulo.username=root
# Accumulo password
test.common.accumulo.password=secret
+# Max memory (in bytes) each batch writer will use to buffer writes
+test.common.accumulo.bw.max.memory.bytes=100000000
+# Max latency (in milliseconds) that each batch writer will buffer data
+test.common.accumulo.bw.max.latency.ms=600000
+# Number of write thread for each batch writer
+test.common.accumulo.bw.num.threads=4
+# Number of threads used by batch scanner
+test.common.accumulo.bs.num.threads=8
+# Number of key/value entries to pull during scan
+test.common.accumulo.scanner.batch.size=1000
# Accumulo keytab
#test.common.accumulo.keytab=
-# Zookeeper connection string
-test.common.zookeepers=localhost:2181
+# HDFS root path. Should match 'fs.defaultFS' property in Hadoop's core-site.xml
+test.common.hdfs.root=hdfs://localhost:10000
+# YARN resource manager hostname. Should match 'yarn.resourcemanager.hostname' property in Hadoop's yarn-site.xml
+test.common.yarn.resource.manager=localhost
# Memory (in MB) given to each container (if running in YARN)
test.common.yarn.container.memory.mb=1024
# Number of cores given to each container (if running in YARN)
test.common.yarn.container.cores=1
-# Max memory (in bytes) each batch writer will use to buffer writes
-test.common.bw.max.memory.bytes=100000000
-# Max latency (in milliseconds) that each batch writer will buffer data
-test.common.bw.max.latency.ms=600000
-# Number of write thread for each batch writer
-test.common.bw.num.threads=4
-# Number of threads used by batch scanner
-test.common.bs.num.threads=8
-# Number of key/value entries to pull during scan
-test.common.scanner.batch.size=1000
+# Zookeeper connection string
+test.common.zookeepers=localhost:2181
###################################
# Continuous ingest test properties
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/28ca75a3/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java b/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java
index cd1f1dc..48266f5 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java
@@ -19,7 +19,7 @@ import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import org.apache.accumulo.core.client.security.tokens.KerberosToken;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.tools.CLI;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
public class TestEnv {
@@ -27,6 +27,7 @@ public class TestEnv {
protected final Properties p;
private Instance instance = null;
private Connector connector = null;
+ private Configuration hadoopConfig = null;
/**
* Creates new test environment using provided properties
@@ -96,15 +97,17 @@ public class TestEnv {
}
public Configuration getHadoopConfiguration() {
- Configuration config = new Configuration();
- config.set("mapreduce.framework.name", "yarn");
- // Setting below are required due to bundled jar breaking default
- // config.
- // See
- // http://stackoverflow.com/questions/17265002/hadoop-no-filesystem-for-scheme-file
- config.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
- config.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
- return config;
+ if (hadoopConfig == null) {
+ hadoopConfig = new Configuration();
+ hadoopConfig.set("fs.defaultFS", getHdfsRoot());
+ // Below is required due to bundled jar breaking default config.
+ // See http://stackoverflow.com/questions/17265002/hadoop-no-filesystem-for-scheme-file
+ hadoopConfig.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
+ hadoopConfig.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
+ hadoopConfig.set("mapreduce.framework.name", "yarn");
+ hadoopConfig.set("yarn.resourcemanager.hostname", getYarnResourceManager());
+ }
+ return hadoopConfig;
}
/**
@@ -135,6 +138,14 @@ public class TestEnv {
return p.getProperty(TestProps.ACCUMULO_INSTANCE);
}
+ public String getHdfsRoot() {
+ return p.getProperty(TestProps.HDFS_ROOT);
+ }
+
+ public String getYarnResourceManager() {
+ return p.getProperty(TestProps.YARN_RESOURCE_MANAGER);
+ }
+
public String getZookeepers() {
return p.getProperty(TestProps.ZOOKEEPERS);
}
@@ -164,9 +175,9 @@ public class TestEnv {
}
public BatchWriterConfig getBatchWriterConfig() {
- int numThreads = Integer.parseInt(p.getProperty(TestProps.BW_NUM_THREADS));
- long maxLatency = Long.parseLong(p.getProperty(TestProps.BW_MAX_LATENCY_MS));
- long maxMemory = Long.parseLong(p.getProperty(TestProps.BW_MAX_MEM_BYTES));
+ int numThreads = Integer.parseInt(p.getProperty(TestProps.ACCUMULO_BW_NUM_THREADS));
+ long maxLatency = Long.parseLong(p.getProperty(TestProps.ACCUMULO_BW_MAX_LATENCY_MS));
+ long maxMemory = Long.parseLong(p.getProperty(TestProps.ACCUMULO_BW_MAX_MEM_BYTES));
BatchWriterConfig config = new BatchWriterConfig();
config.setMaxWriteThreads(numThreads);
@@ -176,6 +187,6 @@ public class TestEnv {
}
public int getScannerBatchSize() {
- return Integer.parseInt(p.getProperty(TestProps.SCANNER_BATCH_SIZE));
+ return Integer.parseInt(p.getProperty(TestProps.ACCUMULO_SCANNER_BATCH_SIZE));
}
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/28ca75a3/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java b/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
index bdc4901..b0927db 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
@@ -42,22 +42,27 @@ public class TestProps {
public static final String ACCUMULO_USERNAME = COMMON + "accumulo.username";
// Accumulo password
public static final String ACCUMULO_PASSWORD = COMMON + "accumulo.password";
+ // Max memory (in bytes) each batch writer will use to buffer writes
+ public static final String ACCUMULO_BW_MAX_MEM_BYTES = COMMON + "accumulo.bw.max.memory.bytes";
+ // Max the maximum time (in ms) each batch writer will buffer data
+ public static final String ACCUMULO_BW_MAX_LATENCY_MS = COMMON + "accumulo.bw.max.latency.ms";
+ // Number of threads each batch writer will use to write data
+ public static final String ACCUMULO_BW_NUM_THREADS = COMMON + "accumulo.bw.num.threads";
+ // Number of thread for each batch scanner
+ public static final String ACCUMULO_BS_NUM_THREADS = COMMON + "accumulo.bw.num.threads";
+ // Number of key/value entries to pull during scan
+ public static final String ACCUMULO_SCANNER_BATCH_SIZE = COMMON + "accumulo.scanner.batch.size";
// Accumulo keytab
public static final String ACCUMULO_KEYTAB = COMMON + "accumulo.keytab";
+ // HDFS root path. Should match 'fs.defaultFS' property in Hadoop's core-site.xml
+ public static final String HDFS_ROOT = COMMON + "hdfs.root";
+ // YARN resource manager hostname. Should match 'yarn.resourcemanager.hostname' property in
+ // Hadoop's yarn-site.xml
+ public static final String YARN_RESOURCE_MANAGER = COMMON + "yarn.resource.manager";
// Memory (in MB) given to each YARN container
public static final String YARN_CONTAINER_MEMORY_MB = COMMON + "yarn.container.memory.mb";
// Number of cores given to each YARN container
public static final String YARN_CONTAINER_CORES = COMMON + "yarn.container.cores";
- // Max memory (in bytes) each batch writer will use to buffer writes
- public static final String BW_MAX_MEM_BYTES = COMMON + "bw.max.memory.bytes";
- // Max the maximum time (in ms) each batch writer will buffer data
- public static final String BW_MAX_LATENCY_MS = COMMON + "bw.max.latency.ms";
- // Number of threads each batch writer will use to write data
- public static final String BW_NUM_THREADS = COMMON + "bw.num.threads";
- // Number of thread for each batch scanner
- public static final String BS_NUM_THREADS = COMMON + "bw.num.threads";
- // Number of key/value entries to pull during scan
- public static final String SCANNER_BATCH_SIZE = COMMON + "scanner.batch.size";
/** Continuous ingest test properties **/
/** Common **/
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/28ca75a3/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java
index 0282c2b..88134a0 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java
@@ -54,7 +54,7 @@ public class ContinuousBatchWalker {
Random r = new Random();
- int scanThreads = Integer.parseInt(props.getProperty(TestProps.BS_NUM_THREADS));
+ int scanThreads = Integer.parseInt(props.getProperty(TestProps.ACCUMULO_BS_NUM_THREADS));
while (true) {
BatchScanner bs = conn.createBatchScanner(env.getAccumuloTableName(), auths, scanThreads);
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/28ca75a3/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
index 8c147a7..30b2191 100644
--- a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
+++ b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.testing.yarn;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.google.common.base.Preconditions;
+import org.apache.accumulo.testing.core.TestEnv;
import org.apache.accumulo.testing.core.TestProps;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.twill.api.ResourceReport;
@@ -36,7 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.FileInputStream;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -125,16 +126,18 @@ public class YarnAccumuloTestRunner {
verifyPath(opts.testProps);
verifyPath(opts.logProps);
+ String jarFileName = Paths.get(opts.jarPath).getFileName().toString();
+
String[] mainArgs = opts.mainArgs.stream().toArray(String[]::new);
- BundledJarRunner.Arguments arguments = new BundledJarRunner.Arguments(opts.jarPath, "/lib", opts.mainClass, mainArgs);
+ BundledJarRunner.Arguments arguments = new BundledJarRunner.Arguments(jarFileName, "/lib",
+ opts.mainClass, mainArgs);
+
+ Properties props = TestProps.loadFromFile(opts.testProps);
+ TestEnv env = new TestEnv(props);
- Properties props = new Properties();
- FileInputStream fis = new FileInputStream(opts.testProps);
- props.load(fis);
- fis.close();
- String zookeepers = props.getProperty(TestProps.ZOOKEEPERS);
+ YarnConfiguration yarnConfig = new YarnConfiguration(env.getHadoopConfiguration());
- final TwillRunnerService twillRunner = new YarnTwillRunnerService(new YarnConfiguration(), zookeepers);
+ TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig, env.getZookeepers());
twillRunner.start();
TwillController controller = twillRunner.prepare(new YarnTestApp(opts, props))