You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/08/18 00:22:48 UTC
sqoop git commit: SQOOP-2491: Sqoop2: Common external JAR handling in
configuration
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 58b2d1c1c -> 094c9c241
SQOOP-2491: Sqoop2: Common external JAR handling in configuration
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/094c9c24
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/094c9c24
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/094c9c24
Branch: refs/heads/sqoop2
Commit: 094c9c241f728da8ee4d034c54bd0ed22b57ca38
Parents: 58b2d1c
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Mon Aug 17 15:21:45 2015 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Mon Aug 17 15:21:45 2015 -0700
----------------------------------------------------------------------
.../org/apache/sqoop/utils/ContextUtils.java | 81 ++++++++++
.../apache/sqoop/utils/TestContextUtils.java | 147 +++++++++++++++++++
.../apache/sqoop/core/SqoopConfiguration.java | 14 +-
3 files changed, 235 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/094c9c24/common/src/main/java/org/apache/sqoop/utils/ContextUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/utils/ContextUtils.java b/common/src/main/java/org/apache/sqoop/utils/ContextUtils.java
new file mode 100644
index 0000000..25d6fcb
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/utils/ContextUtils.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.utils;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Utils to be used ubiquitously for contexts.
+ */
+public class ContextUtils {
+ /**
+ * Return an array of strings for given value or null by default.
+ *
+ * @param value Non-empty value
+ * @return Array of values for given key or null in case of unknown key
+ */
+ public static String[] getArrayOfStrings(String value) {
+ return getArrayOfStrings(value, ":");
+ }
+
+ /**
+ * Return an array of strings for given value or default value.
+ *
+ * @param value Non-empty value
+ * @param separator String separator
+ * @return Array of values for given key or default value in case of unknown key
+ */
+ public static String[] getArrayOfStrings(String value, String separator) {
+ assert !StringUtils.isBlank(value);
+
+ return value.split(separator);
+ }
+
+ private static Set<String> getUniqueStrings(String[] strings) {
+ if (strings == null) {
+ return new HashSet<String>();
+ }
+
+ return new HashSet(Arrays.asList(strings));
+ }
+
+ /**
+ * Return an array of unique strings for given value or null by default.
+ *
+ * @param value Non-empty value
+ * @return Set of unique strings for given value or null in case of unknown key
+ */
+ public static Set<String> getUniqueStrings(String value) {
+ return getUniqueStrings(getArrayOfStrings(value));
+ }
+
+ /**
+ * Return an array of unique strings for given value or default value.
+ *
+ * @param value Non-empty value
+ * @param separator String separator
+ * @return Set of unique strings for given value or default value in case of unknown key
+ */
+ public static Set<String> getUniqueStrings(String value, String separator) {
+ return getUniqueStrings(getArrayOfStrings(value, separator));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/094c9c24/common/src/test/java/org/apache/sqoop/utils/TestContextUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/utils/TestContextUtils.java b/common/src/test/java/org/apache/sqoop/utils/TestContextUtils.java
new file mode 100644
index 0000000..779fc78
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/utils/TestContextUtils.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.utils;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.apache.sqoop.common.MapContext;
+import org.testng.annotations.Test;
+
+import static org.apache.sqoop.utils.ContextUtils.getArrayOfStrings;
+import static org.apache.sqoop.utils.ContextUtils.getUniqueStrings;
+import static org.testng.Assert.assertEquals;
+
+/**
+ * Test class for org.apache.sqoop.utils.ContextUtils
+ */
+public class TestContextUtils {
+ @Test
+ public void testGetArrayOfStrings() {
+ final String DEFAULT_SEPARATOR = ",";
+
+ Map<String, String> options1 = new HashMap<String, String>();
+ options1.put("sqooptest1", "value1:value2");
+ options1.put("sqooptest2", "value1");
+
+ Map<String, String> options2 = new HashMap<String, String>();
+ options2.put("sqooptest1", "value1,value2");
+ options2.put("sqooptest2", "value1");
+
+ MapContext mc1 = new MapContext(options1);
+ MapContext mc2 = new MapContext(options2);
+
+ assertEquals(getArrayOfStrings(mc1.getString("sqooptest1")), new String[]{"value1", "value2"});
+ assertEquals(getArrayOfStrings(mc1.getString("sqooptest2")), new String[]{"value1"});
+
+ assertEquals(getArrayOfStrings(mc2.getString("sqooptest1"), DEFAULT_SEPARATOR), new String[]{"value1", "value2"});
+ assertEquals(getArrayOfStrings(mc2.getString("sqooptest2"), DEFAULT_SEPARATOR), new String[]{"value1"});
+ }
+
+ @Test(expectedExceptions = {AssertionError.class})
+ public void testGetArrayOfStringsNullValue() {
+ Map<String, String> options1 = new HashMap<String, String>();
+ options1.put("sqooptest1", null);
+ MapContext mc1 = new MapContext(options1);
+ getArrayOfStrings(mc1.getString("sqooptest1"));
+ }
+
+ @Test(expectedExceptions = {AssertionError.class})
+ public void testGetArrayOfStringsWithSeparatorNullValue() {
+ Map<String, String> options1 = new HashMap<String, String>();
+ options1.put("sqooptest1", null);
+ MapContext mc1 = new MapContext(options1);
+ getArrayOfStrings(mc1.getString("sqooptest1"), ",");
+ }
+
+ @Test(expectedExceptions = {AssertionError.class})
+ public void testGetArrayOfStringsEmptyValue() {
+ Map<String, String> options1 = new HashMap<String, String>();
+ options1.put("sqooptest1", "");
+ MapContext mc1 = new MapContext(options1);
+ getArrayOfStrings(mc1.getString("sqooptest1"));
+ }
+
+ @Test(expectedExceptions = {AssertionError.class})
+ public void testGetArrayOfStringsWithSeparatorEmptyValue() {
+ Map<String, String> options1 = new HashMap<String, String>();
+ options1.put("sqooptest1", "");
+ MapContext mc1 = new MapContext(options1);
+ getArrayOfStrings(mc1.getString("sqooptest1"), ",");
+ }
+
+ @Test
+ public void testGetUniqueStrings() {
+ final String DEFAULT_SEPARATOR = ",";
+
+ Map<String, String> options1 = new HashMap<String, String>();
+ options1.put("sqooptest1", "value1:value2");
+ options1.put("sqooptest2", "value1");
+ options1.put("sqooptest3", "value1:value2:value1");
+
+ Map<String, String> options2 = new HashMap<String, String>();
+ options2.put("sqooptest1", "value1,value2");
+ options2.put("sqooptest2", "value1");
+ options2.put("sqooptest3", "value1,value2,value1");
+
+ MapContext mc1 = new MapContext(options1);
+ MapContext mc2 = new MapContext(options2);
+
+ assertEquals(getUniqueStrings(mc1.getString("sqooptest1")), new HashSet<String>(Arrays.asList(new String[]{"value1", "value2"})));
+ assertEquals(getUniqueStrings(mc1.getString("sqooptest2")), new HashSet<String>(Arrays.asList(new String[]{"value1"})));
+ assertEquals(getUniqueStrings(mc1.getString("sqooptest3")), new HashSet<String>(Arrays.asList(new String[]{"value1", "value2"})));
+
+ assertEquals(getUniqueStrings(mc2.getString("sqooptest1"), DEFAULT_SEPARATOR), new HashSet<String>(Arrays.asList(new String[]{"value1", "value2"})));
+ assertEquals(getUniqueStrings(mc2.getString("sqooptest2"), DEFAULT_SEPARATOR), new HashSet<String>(Arrays.asList(new String[]{"value1"})));
+ assertEquals(getUniqueStrings(mc2.getString("sqooptest3"), DEFAULT_SEPARATOR), new HashSet<String>(Arrays.asList(new String[]{"value1", "value2"})));
+ }
+
+ @Test(expectedExceptions = {AssertionError.class})
+ public void testGetUniqueStringsNullValue() {
+ Map<String, String> options1 = new HashMap<String, String>();
+ options1.put("sqooptest1", null);
+ MapContext mc1 = new MapContext(options1);
+ getUniqueStrings(mc1.getString("sqooptest1"));
+ }
+
+ @Test(expectedExceptions = {AssertionError.class})
+ public void testGetUniqueStringsWithSeparatorNullValue() {
+ Map<String, String> options1 = new HashMap<String, String>();
+ options1.put("sqooptest1", null);
+ MapContext mc1 = new MapContext(options1);
+ getUniqueStrings(mc1.getString("sqooptest1"), ",");
+ }
+
+ @Test(expectedExceptions = {AssertionError.class})
+ public void testGetUniqueStringsEmptyValue() {
+ Map<String, String> options1 = new HashMap<String, String>();
+ options1.put("sqooptest1", "");
+ MapContext mc1 = new MapContext(options1);
+ getUniqueStrings(mc1.getString("sqooptest1"));
+ }
+
+ @Test(expectedExceptions = {AssertionError.class})
+ public void testGetUniqueStringsWithSeparatorEmptyValue() {
+ Map<String, String> options1 = new HashMap<String, String>();
+ options1.put("sqooptest1", "");
+ MapContext mc1 = new MapContext(options1);
+ getUniqueStrings(mc1.getString("sqooptest1"), ",");
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/094c9c24/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java b/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java
index 58c4c34..49f139c 100644
--- a/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java
+++ b/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java
@@ -24,8 +24,6 @@ import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
-import java.util.Arrays;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -41,6 +39,8 @@ import org.apache.sqoop.common.MapContext;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.error.code.CoreError;
+import static org.apache.sqoop.utils.ContextUtils.getUniqueStrings;
+
/**
* Configuration manager that loads Sqoop configuration.
*/
@@ -230,9 +230,10 @@ public class SqoopConfiguration implements Reconfigurable {
private synchronized void configureClassLoader(String classpathProperty) {
LOG.info("Adding jars to current classloader from property: " + classpathProperty);
- String classpath = getContext().getString(classpathProperty);
+ // CSV URL list separated by ":".
+ String paths = getContext().getString(classpathProperty);
- if (StringUtils.isEmpty(classpath)) {
+ if (StringUtils.isEmpty(paths)) {
LOG.debug("Property " + classpathProperty + " is null or empty. Not adding any extra jars.");
return;
}
@@ -242,11 +243,10 @@ public class SqoopConfiguration implements Reconfigurable {
throw new SqoopException(CoreError.CORE_0009, "No thread context classloader to override.");
}
- // CSV URL list separated by ":".
- Set<String> paths = new HashSet(Arrays.asList(classpath.split(":")));
+ Set<String> pathSet = getUniqueStrings(paths);
List<URL> urls = new LinkedList<URL>();
- for (String path : paths) {
+ for (String path : pathSet) {
try {
LOG.debug("Found jar in path: " + path);
URL url = new File(path).toURI().toURL();