You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2013/01/10 18:01:47 UTC
git commit: SQOOP-796: Unable to use new Hadoop environment variables
Updated Branches:
refs/heads/trunk fd0143b5a -> 79f941b7e
SQOOP-796: Unable to use new Hadoop environment variables
(Sean Mackrory via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/79f941b7
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/79f941b7
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/79f941b7
Branch: refs/heads/trunk
Commit: 79f941b7e14df9b555084a7eee05980221d05f9e
Parents: fd0143b
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Thu Jan 10 09:01:06 2013 -0800
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Thu Jan 10 09:01:06 2013 -0800
----------------------------------------------------------------------
bin/configure-sqoop | 29 ++++++++++++---
bin/sqoop | 2 +-
conf/sqoop-env-template.sh | 5 ++-
src/docs/man/common-args.txt | 7 +++-
src/docs/man/sqoop.txt | 10 ++++--
src/docs/user/common-args.txt | 2 +-
src/docs/user/help.txt | 3 +-
src/docs/user/tools.txt | 27 ++++++++------
src/java/org/apache/sqoop/SqoopOptions.java | 16 ++++----
.../org/apache/sqoop/orm/CompilationManager.java | 21 ++++++-----
src/java/org/apache/sqoop/tool/BaseSqoopTool.java | 10 ++++--
src/scripts/run-perftest.sh | 2 +-
12 files changed, 86 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/bin/configure-sqoop
----------------------------------------------------------------------
diff --git a/bin/configure-sqoop b/bin/configure-sqoop
index ee1dd7b..61ff3f2 100755
--- a/bin/configure-sqoop
+++ b/bin/configure-sqoop
@@ -40,17 +40,33 @@ fi
# Find paths to our dependency systems. If they are unset, use CDH defaults.
-if [ -z "${HADOOP_HOME}" ]; then
- HADOOP_HOME=/usr/lib/hadoop
+if [ -z "${HADOOP_COMMON_HOME}" ]; then
+ if [ -n "${HADOOP_HOME}" ]; then
+ HADOOP_COMMON_HOME=${HADOOP_HOME}
+ else
+ HADOOP_COMMON_HOME=/usr/lib/hadoop
+ fi
+fi
+if [ -z "${HADOOP_MAPRED_HOME}" ]; then
+ if [ -n "${HADOOP_HOME}" ]; then
+ HADOOP_MAPRED_HOME=${HADOOP_HOME}
+ else
+ HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce
+ fi
fi
if [ -z "${HBASE_HOME}" ]; then
HBASE_HOME=/usr/lib/hbase
fi
# Check: If we can't find our dependencies, give up here.
-if [ ! -d "${HADOOP_HOME}" ]; then
- echo "Error: $HADOOP_HOME does not exist!"
- echo 'Please set $HADOOP_HOME to the root of your Hadoop installation.'
+if [ ! -d "${HADOOP_COMMON_HOME}" ]; then
+ echo "Error: $HADOOP_COMMON_HOME does not exist!"
+ echo 'Please set $HADOOP_COMMON_HOME to the root of your Hadoop installation.'
+ exit 1
+fi
+if [ ! -d "${HADOOP_MAPRED_HOME}" ]; then
+ echo "Error: $HADOOP_MAPRED_HOME does not exist!"
+ echo 'Please set $HADOOP_MAPRED_HOME to the root of your Hadoop MapReduce installation.'
exit 1
fi
@@ -117,6 +133,7 @@ export SQOOP_CLASSPATH
export SQOOP_CONF_DIR
export SQOOP_JAR_DIR
export HADOOP_CLASSPATH
-export HADOOP_HOME
+export HADOOP_COMMON_HOME
+export HADOOP_MAPRED_HOME
export HBASE_HOME
http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/bin/sqoop
----------------------------------------------------------------------
diff --git a/bin/sqoop b/bin/sqoop
index bc69083..0595714 100755
--- a/bin/sqoop
+++ b/bin/sqoop
@@ -98,4 +98,4 @@ bin=`dirname ${prgm}`
bin=`cd ${bin} && pwd`
source ${bin}/configure-sqoop "${bin}"
-exec ${HADOOP_HOME}/bin/hadoop org.apache.sqoop.Sqoop "$@"
+exec ${HADOOP_COMMON_HOME}/bin/hadoop org.apache.sqoop.Sqoop "$@"
http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/conf/sqoop-env-template.sh
----------------------------------------------------------------------
diff --git a/conf/sqoop-env-template.sh b/conf/sqoop-env-template.sh
index 0294f17..90bde78 100644
--- a/conf/sqoop-env-template.sh
+++ b/conf/sqoop-env-template.sh
@@ -20,7 +20,10 @@
# Set Hadoop-specific environment variables here.
#Set path to where bin/hadoop is available
-#export HADOOP_HOME=
+#export HADOOP_COMMON_HOME=
+
+#Set path to where hadoop-*-core.jar is available
+#export HADOOP_MAPRED_HOME=
#set the path to where bin/hbase is available
#export HBASE_HOME=
http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/docs/man/common-args.txt
----------------------------------------------------------------------
diff --git a/src/docs/man/common-args.txt b/src/docs/man/common-args.txt
index 129f5d1..cf9c0c3 100644
--- a/src/docs/man/common-args.txt
+++ b/src/docs/man/common-args.txt
@@ -33,8 +33,8 @@ Database connection and common options
--connection-param-file (filename)::
Optional properties file that provides connection parameters
---hadoop-home (dir)::
- Override $HADOOP_HOME
+--hadoop-mapred-home (dir)::
+ Override $HADOOP_MAPRED_HOME
--help::
Print usage instructions
@@ -51,3 +51,6 @@ Database connection and common options
--verbose::
Print more information while working
+
+--hadoop-home (dir)::
+ Deprecated. Override $HADOOP_HOME
http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/docs/man/sqoop.txt
----------------------------------------------------------------------
diff --git a/src/docs/man/sqoop.txt b/src/docs/man/sqoop.txt
index 44b4f4e..febe827 100644
--- a/src/docs/man/sqoop.txt
+++ b/src/docs/man/sqoop.txt
@@ -43,9 +43,13 @@ JAVA_HOME::
e.g., +/usr/java/default+. Hadoop (and Sqoop) requires Sun Java 1.6 which
can be downloaded from http://java.sun.com.
-HADOOP_HOME::
- The location of the Hadoop jar files. If you installed Hadoop via RPM
- or DEB, these are in +/usr/lib/hadoop-20+.
+HADOOP_COMMON_HOME::
+ The location of the Hadoop Common files (specifically the bin/hadoop executable).
+ If you installed Hadoop via RPM or DEB, this is in +/usr/lib/hadoop+.
+
+HADOOP_MAPRED_HOME::
+ The location of the Hadoop MapReduce files (specifically the JAR files).
+ If you installed Hadoop via RPM or DEB, this is in +/usr/lib/hadoop-mapreduce+.
HIVE_HOME::
If you are performing a Hive import, you must identify the location of
http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/docs/user/common-args.txt
----------------------------------------------------------------------
diff --git a/src/docs/user/common-args.txt b/src/docs/user/common-args.txt
index 7c69886..0554f81 100644
--- a/src/docs/user/common-args.txt
+++ b/src/docs/user/common-args.txt
@@ -27,7 +27,7 @@ Argument Description
use
+\--driver <class-name>+ Manually specify JDBC driver class\
to use
-+\--hadoop-home <dir>+ Override $HADOOP_HOME
++\--hadoop-mapred-home <dir>+ Override $HADOOP_MAPRED_HOME
+\--help+ Print usage instructions
+-P+ Read password from console
+\--password <password>+ Set authentication password
http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/docs/user/help.txt
----------------------------------------------------------------------
diff --git a/src/docs/user/help.txt b/src/docs/user/help.txt
index 11b4f0c..24fbddc 100644
--- a/src/docs/user/help.txt
+++ b/src/docs/user/help.txt
@@ -68,12 +68,13 @@ Common arguments:
--connect <jdbc-uri> Specify JDBC connect string
--connection-manager <class-name> Specify connection manager class to use
--driver <class-name> Manually specify JDBC driver class to use
- --hadoop-home <dir> Override $HADOOP_HOME
+ --hadoop-mapred-home <dir> Override $HADOOP_MAPRED_HOME
--help Print usage instructions
-P Read password from console
--password <password> Set authentication password
--username <username> Set authentication username
--verbose Print more information while working
+ --hadoop-home <dir> Deprecated. Override $HADOOP_HOME
Import control arguments:
--as-avrodatafile Imports data to Avro Data Files
http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/docs/user/tools.txt
----------------------------------------------------------------------
diff --git a/src/docs/user/tools.txt b/src/docs/user/tools.txt
index ce83a24..96bf777 100644
--- a/src/docs/user/tools.txt
+++ b/src/docs/user/tools.txt
@@ -25,10 +25,9 @@ tool you want to use and the arguments that control the tool.
If Sqoop is compiled from its own source, you can run Sqoop without a formal
installation process by running the +bin/sqoop+ program. Users
-of a packaged deployment of Sqoop (such as an RPM shipped with Cloudera's
-Distribution for Hadoop) will see this program installed as +/usr/bin/sqoop+.
-The remainder of this documentation will refer to this program as
-+sqoop+. For example:
+of a packaged deployment of Sqoop (such as an RPM shipped with Apache Bigtop)
+will see this program installed as +/usr/bin/sqoop+. The remainder of this
+documentation will refer to this program as +sqoop+. For example:
----
$ sqoop tool-name [tool-arguments]
@@ -87,24 +86,29 @@ You invoke Sqoop through the program launch capability provided by
Hadoop. The +sqoop+ command-line program is a wrapper which runs the
+bin/hadoop+ script shipped with Hadoop. If you have multiple
installations of Hadoop present on your machine, you can select the
-Hadoop installation by setting the +$HADOOP_HOME+ environment
-variable.
+Hadoop installation by setting the +$HADOOP_COMMON_HOME+ and
++$HADOOP_MAPRED_HOME+ environment variables.
For example:
----
-$ HADOOP_HOME=/path/to/some/hadoop sqoop import --arguments...
+$ HADOOP_COMMON_HOME=/path/to/some/hadoop \
+ HADOOP_MAPRED_HOME=/path/to/some/hadoop-mapreduce \
+ sqoop import --arguments...
----
or:
----
-$ export HADOOP_HOME=/some/path/to/hadoop
+$ export HADOOP_COMMON_HOME=/some/path/to/hadoop
+$ export HADOOP_MAPRED_HOME=/some/path/to/hadoop-mapreduce
$ sqoop import --arguments...
-----
-If +$HADOOP_HOME+ is not set, Sqoop will use the default installation
-location for Cloudera's Distribution for Hadoop, +/usr/lib/hadoop+.
+If either of these variables are not set, Sqoop will fall back to
++$HADOOP_HOME+. If it is not set either, Sqoop will use the default
+installation locations for Apache Bigtop, +/usr/lib/hadoop+ and
++/usr/lib/hadoop-mapreduce+, respectively.
The active Hadoop configuration is loaded from +$HADOOP_HOME/conf/+,
unless the +$HADOOP_CONF_DIR+ environment variable is set.
@@ -126,12 +130,13 @@ Common arguments:
--connect <jdbc-uri> Specify JDBC connect string
--connect-manager <jdbc-uri> Specify connection manager class to use
--driver <class-name> Manually specify JDBC driver class to use
- --hadoop-home <dir> Override $HADOOP_HOME
+ --hadoop-mapred-home <dir>+ Override $HADOOP_MAPRED_HOME
--help Print usage instructions
-P Read password from console
--password <password> Set authentication password
--username <username> Set authentication username
--verbose Print more information while working
+ --hadoop-home <dir>+ Deprecated. Override $HADOOP_HOME
[...]
http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/java/org/apache/sqoop/SqoopOptions.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java
index 3e0ec3e..b0fdfa0 100644
--- a/src/java/org/apache/sqoop/SqoopOptions.java
+++ b/src/java/org/apache/sqoop/SqoopOptions.java
@@ -119,7 +119,7 @@ public class SqoopOptions implements Cloneable {
// explicitly set by the user (false). If the former, disregard any value
// for jarOutputDir saved in the metastore.
@StoredAsProperty("codegen.auto.compile.dir") private boolean jarDirIsAuto;
- private String hadoopHome; // not serialized to metastore.
+ private String hadoopMapRedHome; // not serialized to metastore.
@StoredAsProperty("db.split.column") private String splitByCol;
@StoredAsProperty("db.where.clause") private String whereClause;
@StoredAsProperty("db.query") private String sqlQuery;
@@ -775,7 +775,7 @@ public class SqoopOptions implements Cloneable {
private void initDefaults(Configuration baseConfiguration) {
// first, set the true defaults if nothing else happens.
// default action is to run the full pipeline.
- this.hadoopHome = System.getenv("HADOOP_HOME");
+ this.hadoopMapRedHome = System.getenv("HADOOP_MAPRED_HOME");
this.hiveHome = getHiveHomeDefault();
@@ -1253,15 +1253,15 @@ public class SqoopOptions implements Cloneable {
}
/**
- * Return the value of $HADOOP_HOME.
- * @return $HADOOP_HOME, or null if it's not set.
+ * Return the value of $HADOOP_MAPRED_HOME.
+ * @return $HADOOP_MAPRED_HOME, or null if it's not set.
*/
- public String getHadoopHome() {
- return hadoopHome;
+ public String getHadoopMapRedHome() {
+ return hadoopMapRedHome;
}
- public void setHadoopHome(String home) {
- this.hadoopHome = home;
+ public void setHadoopMapRedHome(String home) {
+ this.hadoopMapRedHome = home;
}
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/java/org/apache/sqoop/orm/CompilationManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/orm/CompilationManager.java b/src/java/org/apache/sqoop/orm/CompilationManager.java
index 26adcbf..92effb5 100644
--- a/src/java/org/apache/sqoop/orm/CompilationManager.java
+++ b/src/java/org/apache/sqoop/orm/CompilationManager.java
@@ -71,28 +71,29 @@ public class CompilationManager {
}
/**
- * locate the hadoop-*-core.jar in $HADOOP_HOME or --hadoop-home.
+ * locate the hadoop-*-core.jar in $HADOOP_MAPRED_HOME or
+ * --hadoop-mapred-home.
* If that doesn't work, check our classpath.
* @return the filename of the hadoop-*-core.jar file.
*/
private String findHadoopCoreJar() {
- String hadoopHome = options.getHadoopHome();
+ String hadoopMapRedHome = options.getHadoopMapRedHome();
- if (null == hadoopHome) {
- LOG.info("$HADOOP_HOME is not set");
+ if (null == hadoopMapRedHome) {
+ LOG.info("$HADOOP_MAPRED_HOME is not set");
return Jars.getJarPathForClass(JobConf.class);
}
- if (!hadoopHome.endsWith(File.separator)) {
- hadoopHome = hadoopHome + File.separator;
+ if (!hadoopMapRedHome.endsWith(File.separator)) {
+ hadoopMapRedHome = hadoopMapRedHome + File.separator;
}
- File hadoopHomeFile = new File(hadoopHome);
- LOG.info("HADOOP_HOME is " + hadoopHomeFile.getAbsolutePath());
- File [] entries = hadoopHomeFile.listFiles();
+ File hadoopMapRedHomeFile = new File(hadoopMapRedHome);
+ LOG.info("HADOOP_MAPRED_HOME is " + hadoopMapRedHomeFile.getAbsolutePath());
+ File [] entries = hadoopMapRedHomeFile.listFiles();
if (null == entries) {
- LOG.warn("HADOOP_HOME appears empty or missing");
+ LOG.warn("HADOOP_MAPRED_HOME appears empty or missing");
return Jars.getJarPathForClass(JobConf.class);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index ea8a3f4..b4b2213 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -80,6 +80,7 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool {
public static final String SPLIT_BY_ARG = "split-by";
public static final String WHERE_ARG = "where";
public static final String HADOOP_HOME_ARG = "hadoop-home";
+ public static final String HADOOP_MAPRED_HOME_ARG = "hadoop-mapred-home";
public static final String HIVE_HOME_ARG = "hive-home";
public static final String WAREHOUSE_DIR_ARG = "warehouse-dir";
public static final String TARGET_DIR_ARG = "target-dir";
@@ -385,8 +386,8 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool {
.create(PASSWORD_PROMPT_ARG));
commonOpts.addOption(OptionBuilder.withArgName("dir")
- .hasArg().withDescription("Override $HADOOP_HOME")
- .withLongOpt(HADOOP_HOME_ARG)
+ .hasArg().withDescription("Override $HADOOP_MAPRED_HOME_ARG")
+ .withLongOpt(HADOOP_MAPRED_HOME_ARG)
.create());
// misc (common)
@@ -750,7 +751,10 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool {
}
if (in.hasOption(HADOOP_HOME_ARG)) {
- out.setHadoopHome(in.getOptionValue(HADOOP_HOME_ARG));
+ out.setHadoopMapRedHome(in.getOptionValue(HADOOP_HOME_ARG));
+ }
+ if (in.hasOption(HADOOP_MAPRED_HOME_ARG)) {
+ out.setHadoopMapRedHome(in.getOptionValue(HADOOP_MAPRED_HOME_ARG));
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/79f941b7/src/scripts/run-perftest.sh
----------------------------------------------------------------------
diff --git a/src/scripts/run-perftest.sh b/src/scripts/run-perftest.sh
index 2a4ba21..55beaa7 100755
--- a/src/scripts/run-perftest.sh
+++ b/src/scripts/run-perftest.sh
@@ -37,5 +37,5 @@ source ${SQOOP_HOME}/bin/configure-sqoop "${bin}"
PERFTEST_CLASSES=${SQOOP_HOME}/build/perftest/classes
export HADOOP_CLASSPATH=${PERFTEST_CLASSES}:${SQOOP_JAR}:${HADOOP_CLASSPATH}
-${HADOOP_HOME}/bin/hadoop "$@"
+${HADOOP_COMMON_HOME}/bin/hadoop "$@"