You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by va...@apache.org on 2018/07/20 08:56:05 UTC
sqoop git commit: SQOOP-3346: Upgrade Hadoop version to 2.8.0
Repository: sqoop
Updated Branches:
refs/heads/trunk 739bbce48 -> b3e941be0
SQOOP-3346: Upgrade Hadoop version to 2.8.0
(Boglarka Egyed via Szabolcs Vasas)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/b3e941be
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/b3e941be
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/b3e941be
Branch: refs/heads/trunk
Commit: b3e941be050d615f6aae5b4fafb9e0754c8c6a72
Parents: 739bbce
Author: Szabolcs Vasas <va...@apache.org>
Authored: Fri Jul 20 10:54:40 2018 +0200
Committer: Szabolcs Vasas <va...@apache.org>
Committed: Fri Jul 20 10:54:40 2018 +0200
----------------------------------------------------------------------
ivy/libraries.properties | 2 +-
.../sqoop/config/ConfigurationConstants.java | 15 +++----------
.../sqoop/config/ConfigurationHelper.java | 23 +++++---------------
.../org/apache/sqoop/mapreduce/JobBase.java | 4 +---
src/java/org/apache/sqoop/tool/ImportTool.java | 5 ++---
src/test/org/apache/sqoop/TestSqoopOptions.java | 1 +
.../sqlserver/SQLServerParseMethodsTest.java | 2 --
.../org/apache/sqoop/orm/TestParseMethods.java | 4 ----
8 files changed, 13 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/ivy/libraries.properties
----------------------------------------------------------------------
diff --git a/ivy/libraries.properties b/ivy/libraries.properties
index c506ca8..d679b12 100644
--- a/ivy/libraries.properties
+++ b/ivy/libraries.properties
@@ -55,7 +55,7 @@ accumulo.version=1.6.2
slf4j.version=1.7.7
-hadoop.version=2.6.0
+hadoop.version=2.8.0
hbase.version=1.2.4
hcatalog.version=1.2.1
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/java/org/apache/sqoop/config/ConfigurationConstants.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/config/ConfigurationConstants.java b/src/java/org/apache/sqoop/config/ConfigurationConstants.java
index 666852c..3724f25 100644
--- a/src/java/org/apache/sqoop/config/ConfigurationConstants.java
+++ b/src/java/org/apache/sqoop/config/ConfigurationConstants.java
@@ -55,23 +55,14 @@ public final class ConfigurationConstants {
"mapred.reduce.tasks.speculative.execution";
/**
- * The Configuration property identifying the job tracker address.
- */
- public static final String PROP_MAPRED_JOB_TRACKER_ADDRESS =
- "mapred.job.tracker";
-
- /**
- * The Configuration property identifying the job tracker address (new).
- */
- public static final String PROP_MAPREDUCE_JOB_TRACKER_ADDRESS =
- "mapreduce.jobtracker.address";
-
- /**
* The Configuration property identifying the framework name. If set to YARN
* then we will not be in local mode.
*/
public static final String PROP_MAPREDUCE_FRAMEWORK_NAME =
"mapreduce.framework.name";
+
+ public static final String MAPREDUCE_FRAMEWORK_LOCAL = "local";
+
/**
* The group name of task counters.
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/java/org/apache/sqoop/config/ConfigurationHelper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/config/ConfigurationHelper.java b/src/java/org/apache/sqoop/config/ConfigurationHelper.java
index fb2ab03..5a3d8ff 100644
--- a/src/java/org/apache/sqoop/config/ConfigurationHelper.java
+++ b/src/java/org/apache/sqoop/config/ConfigurationHelper.java
@@ -31,6 +31,9 @@ import org.apache.sqoop.mapreduce.db.DBConfiguration;
import org.apache.hadoop.util.ReflectionUtils;
+import static org.apache.sqoop.config.ConfigurationConstants.MAPREDUCE_FRAMEWORK_LOCAL;
+import static org.apache.sqoop.config.ConfigurationConstants.PROP_MAPREDUCE_FRAMEWORK_NAME;
+
/**
* This class provides static helper methods that allow access and manipulation
* of job configuration. It is convenient to keep such access in one place in
@@ -121,13 +124,6 @@ public final class ConfigurationHelper {
}
/**
- * Sets the Jobtracker address to use for a job.
- */
- public static void setJobtrackerAddr(Configuration conf, String addr) {
- conf.set(ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS, addr);
- }
-
- /**
* @return the Configuration property identifying a DBWritable to use.
*/
public static String getDbInputClassProperty() {
@@ -233,17 +229,8 @@ public final class ConfigurationHelper {
return config.getInt(ConfigurationConstants.PROP_SPLIT_LIMIT, -1);
}
public static boolean isLocalJobTracker(Configuration conf) {
- // If framework is set to YARN, then we can't be running in local mode
- if ("yarn".equalsIgnoreCase(conf
- .get(ConfigurationConstants.PROP_MAPREDUCE_FRAMEWORK_NAME))) {
- return false;
- }
- String jtAddr = conf
- .get(ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS);
- String jtAddr2 = conf
- .get(ConfigurationConstants.PROP_MAPREDUCE_JOB_TRACKER_ADDRESS);
- return (jtAddr != null && jtAddr.equals("local"))
- || (jtAddr2 != null && jtAddr2.equals("local"));
+ String frameworkName = conf.get(PROP_MAPREDUCE_FRAMEWORK_NAME);
+ return MAPREDUCE_FRAMEWORK_LOCAL.equals(frameworkName);
}
private ConfigurationHelper() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/java/org/apache/sqoop/mapreduce/JobBase.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/JobBase.java b/src/java/org/apache/sqoop/mapreduce/JobBase.java
index 6d1e049..f7b1b46 100644
--- a/src/java/org/apache/sqoop/mapreduce/JobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/JobBase.java
@@ -272,9 +272,7 @@ public class JobBase {
protected void loadJars(Configuration conf, String ormJarFile,
String tableClassName) throws IOException {
- boolean isLocal = "local".equals(conf.get("mapreduce.jobtracker.address"))
- || "local".equals(conf.get("mapred.job.tracker"));
- if (isLocal) {
+ if (ConfigurationHelper.isLocalJobTracker(conf)) {
// If we're using the LocalJobRunner, then instead of using the compiled
// jar file as the job source, we're running in the current thread. Push
// on another classloader that loads from that jar in addition to
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/java/org/apache/sqoop/tool/ImportTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java
index f7310b9..ccded65 100644
--- a/src/java/org/apache/sqoop/tool/ImportTool.java
+++ b/src/java/org/apache/sqoop/tool/ImportTool.java
@@ -42,6 +42,7 @@ import org.apache.sqoop.SqoopOptions;
import org.apache.sqoop.SqoopOptions.InvalidOptionsException;
import org.apache.sqoop.cli.RelatedOptions;
import org.apache.sqoop.cli.ToolOptions;
+import org.apache.sqoop.config.ConfigurationHelper;
import org.apache.sqoop.hive.HiveClient;
import org.apache.sqoop.hive.HiveClientFactory;
import org.apache.sqoop.manager.ImportJobContext;
@@ -118,9 +119,7 @@ public class ImportTool extends BaseSqoopTool {
private void loadJars(Configuration conf, String ormJarFile,
String tableClassName) throws IOException {
- boolean isLocal = "local".equals(conf.get("mapreduce.jobtracker.address"))
- || "local".equals(conf.get("mapred.job.tracker"));
- if (isLocal) {
+ if (ConfigurationHelper.isLocalJobTracker(conf)) {
// If we're using the LocalJobRunner, then instead of using the compiled
// jar file as the job source, we're running in the current thread. Push
// on another classloader that loads from that jar in addition to
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/test/org/apache/sqoop/TestSqoopOptions.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/TestSqoopOptions.java b/src/test/org/apache/sqoop/TestSqoopOptions.java
index ba4a4d4..d0591ad 100644
--- a/src/test/org/apache/sqoop/TestSqoopOptions.java
+++ b/src/test/org/apache/sqoop/TestSqoopOptions.java
@@ -858,6 +858,7 @@ public class TestSqoopOptions {
@Test
public void testSqoopOptionsCloneIsEqual() throws Exception {
SqoopOptions options = createSqoopOptionsFilledWithRandomData();
+ options.getConf().setAllowNullValueProperties(false); // always false in cloned conf
SqoopOptions clonedOptions = (SqoopOptions) options.clone();
assertThat(options).isEqualToComparingFieldByFieldRecursively(clonedOptions);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java
index 833ebe8..fbd8d96 100644
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java
@@ -172,8 +172,6 @@ public class SQLServerParseMethodsTest extends ImportJobTestCase {
// Tell the job what class we're testing.
job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName);
- // use local mode in the same JVM.
- ConfigurationHelper.setJobtrackerAddr(job, "local");
job.set("fs.default.name", "file:///");
String warehouseDir = getWarehouseDir();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/test/org/apache/sqoop/orm/TestParseMethods.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/orm/TestParseMethods.java b/src/test/org/apache/sqoop/orm/TestParseMethods.java
index 46bb52d..0d34f5a 100644
--- a/src/test/org/apache/sqoop/orm/TestParseMethods.java
+++ b/src/test/org/apache/sqoop/orm/TestParseMethods.java
@@ -135,8 +135,6 @@ public class TestParseMethods extends ImportJobTestCase {
// Tell the job what class we're testing.
job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName);
- // use local mode in the same JVM.
- ConfigurationHelper.setJobtrackerAddr(job, "local");
if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
}
@@ -263,8 +261,6 @@ public class TestParseMethods extends ImportJobTestCase {
job.set(ExplicitSetMapper.SET_COL_KEY, BASE_COL_NAME + "0");
job.set(ExplicitSetMapper.SET_VAL_KEY, "this-is-a-test");
- // use local mode in the same JVM.
- ConfigurationHelper.setJobtrackerAddr(job, "local");
if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
}