You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ab...@apache.org on 2015/03/11 22:13:46 UTC
sqoop git commit: SQOOP-1068: Sqoop2: Add integration tests for real
cluster
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 1a4293df7 -> 3f618c917
SQOOP-1068: Sqoop2: Add integration tests for real cluster
(Syed Hashmi via Abraham Elmahrek)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/3f618c91
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/3f618c91
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/3f618c91
Branch: refs/heads/sqoop2
Commit: 3f618c917695ab45e09e4e432c9e58f5df466e67
Parents: 1a4293d
Author: Abraham Elmahrek <ab...@apache.org>
Authored: Wed Mar 11 14:12:19 2015 -0700
Committer: Abraham Elmahrek <ab...@apache.org>
Committed: Wed Mar 11 14:12:19 2015 -0700
----------------------------------------------------------------------
.../sqoop/test/hadoop/HadoopLocalRunner.java | 9 ++-
.../test/hadoop/HadoopMiniClusterRunner.java | 19 +++--
.../test/hadoop/HadoopRealClusterRunner.java | 84 ++++++++++++++++++++
.../apache/sqoop/test/hadoop/HadoopRunner.java | 30 ++++---
4 files changed, 117 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3f618c91/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java
index 44465b4..140d0f9 100644
--- a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java
+++ b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java
@@ -21,14 +21,15 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.sqoop.test.utils.HdfsUtils;
/**
- * Represents a local cluster.
- * It uses an unchanged Configuration object.
- * HadoopRunner implementation that is using LocalJobRunner for executing mapreduce jobs and local filesystem instead of HDFS.
+ * Represents a local cluster. It uses an unchanged Configuration object.
+ * HadoopRunner implementation that is using LocalJobRunner for executing
+ * mapreduce jobs and local filesystem instead of HDFS.
*/
public class HadoopLocalRunner extends HadoopRunner {
@Override
- public Configuration prepareConfiguration(Configuration conf) {
+ public Configuration prepareConfiguration(Configuration conf)
+ throws Exception {
return conf;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3f618c91/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java
index b06dcab..2c0c4e6 100644
--- a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java
+++ b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java
@@ -24,13 +24,13 @@ import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.log4j.Logger;
/**
- * Represents a minicluster setup.
- * It creates a configuration object and mutates it.
- * Clients that need to connect to the miniclusters should use
- * the provided configuration object.
+ * Represents a minicluster setup. It creates a configuration object and mutates
+ * it. Clients that need to connect to the miniclusters should use the provided
+ * configuration object.
*/
public class HadoopMiniClusterRunner extends HadoopRunner {
- private static final Logger LOG = Logger.getLogger(HadoopMiniClusterRunner.class);
+ private static final Logger LOG = Logger
+ .getLogger(HadoopMiniClusterRunner.class);
/**
* Hadoop HDFS cluster
@@ -43,7 +43,8 @@ public class HadoopMiniClusterRunner extends HadoopRunner {
protected MiniMRCluster mrCluster;
@Override
- public Configuration prepareConfiguration(Configuration config) {
+ public Configuration prepareConfiguration(Configuration config)
+ throws Exception {
config.set("dfs.block.access.token.enable", "false");
config.set("dfs.permissions", "true");
config.set("hadoop.security.authentication", "simple");
@@ -51,7 +52,8 @@ public class HadoopMiniClusterRunner extends HadoopRunner {
config.set("mapred.tasktracker.reduce.tasks.maximum", "1");
config.set("mapred.submit.replication", "1");
config.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
- config.set("yarn.application.classpath", System.getProperty("java.class.path"));
+ config.set("yarn.application.classpath",
+ System.getProperty("java.class.path"));
return config;
}
@@ -75,7 +77,8 @@ public class HadoopMiniClusterRunner extends HadoopRunner {
// Start MR server
LOG.info("Starting MR cluster");
- mrCluster = new MiniMRCluster(0, 0, 1, dfsCluster.getFileSystem().getUri().toString(), 1, null, null, null, new JobConf(config));
+ mrCluster = new MiniMRCluster(0, 0, 1, dfsCluster.getFileSystem().getUri()
+ .toString(), 1, null, null, null, new JobConf(config));
LOG.info("Started MR cluster");
config = prepareConfiguration(mrCluster.createJobConf());
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3f618c91/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java
new file mode 100644
index 0000000..eabdf0b
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.test.hadoop;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/*
+* This class enables running tests against a real cluster.
+* From the command line, it expects "sqoop.hadoop.config.path"
+* variable to point to directory containing cluster config files,
+* otherwise it tries loading from default location: /etc/hadoop/conf.
+*/
+public class HadoopRealClusterRunner extends HadoopRunner {
+
+ private static final Logger LOG = Logger
+ .getLogger(HadoopRealClusterRunner.class);
+
+ /*
+ * This method loads config files for real cluster.
+ * core-site.xml, mapred-site.xml and hdfs-site.xml are mandatory
+ * while yarn-site.xml is optional to let tests execute against old
+ * (non-yarn based) M/R clusters.
+ */
+ @Override
+ public Configuration prepareConfiguration(Configuration config)
+ throws Exception {
+ String configPath = System.getProperty(
+ "sqoop.hadoop.config.path", "/etc/hadoop/conf");
+ LOG.debug("Config path is: " + configPath);
+
+ File dir = new File(configPath);
+ String [] files = dir.list(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.endsWith("-site.xml");
+ }
+ });
+
+ if(files == null) {
+ throw new FileNotFoundException("Hadoop config files not found: " + configPath);
+ }
+
+ // Add each config file to configuration object
+ for (String file : files) {
+ LOG.info("Found hadoop configuration file " + file);
+ config.addResource(new Path(configPath, file));
+ }
+ return config;
+ }
+
+ @Override
+ public void start() throws Exception {
+ // Do nothing
+
+ }
+
+ @Override
+ public void stop() throws Exception {
+ // Do nothing
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3f618c91/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java
index 2516ff1..cb4e384 100644
--- a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java
+++ b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java
@@ -22,15 +22,17 @@ import org.apache.sqoop.test.utils.HdfsUtils;
/**
* Hadoop cluster runner for testing purpose.
- *
- * Runner provides methods for bootstrapping and using Hadoop cluster.
- * This abstract implementation is agnostic about in what mode Hadoop is running.
- * Each mode will have it's own concrete implementation (for example LocalJobRunner, MiniCluster or Real existing cluster).
+ *
+ * Runner provides methods for bootstrapping and using Hadoop cluster. This
+ * abstract implementation is agnostic about in what mode Hadoop is running.
+ * Each mode will have it's own concrete implementation (for example
+ * LocalJobRunner, MiniCluster or Real existing cluster).
*/
public abstract class HadoopRunner {
/**
- * Temporary path that can be used as a root for other directories storing various data like logs or stored HDFS files.
+ * Temporary path that can be used as a root for other directories storing
+ * various data like logs or stored HDFS files.
*/
private String temporaryPath;
@@ -40,12 +42,14 @@ public abstract class HadoopRunner {
protected Configuration config = null;
/**
- * Prepare configuration object.
- * This method should be called once before the start method is called.
+ * Prepare configuration object. This method should be called once before the
+ * start method is called.
*
- * @param config is the configuration object to prepare.
+ * @param config
+ * is the configuration object to prepare.
*/
- abstract public Configuration prepareConfiguration(Configuration config);
+ abstract public Configuration prepareConfiguration(Configuration config)
+ throws Exception;
/**
* Start hadoop cluster.
@@ -89,8 +93,8 @@ public abstract class HadoopRunner {
}
/**
- * Return directory on local filesystem where logs and other
- * data generated by the Hadoop Cluster should be stored.
+ * Return directory on local filesystem where logs and other data generated by
+ * the Hadoop Cluster should be stored.
*
* @return
*/
@@ -99,8 +103,8 @@ public abstract class HadoopRunner {
}
/**
- * Return directory on local filesystem where logs and other
- * data generated by the Hadoop Cluster should be stored.
+ * Return directory on local filesystem where logs and other data generated by
+ * the Hadoop Cluster should be stored.
*
* @return
*/