You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2014/12/15 17:29:14 UTC
sqoop git commit: SQOOP-1898: Change the API of getJars to use SET
than LIST to avoid duplicate jars
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 5abcdd4d6 -> 80662d6d4
SQOOP-1898: Change the API of getJars to use SET than LIST to avoid duplicate jars
(Veena Basavaraj via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/80662d6d
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/80662d6d
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/80662d6d
Branch: refs/heads/sqoop2
Commit: 80662d6d4590b19a905a6c0d51c4580f3219c1c5
Parents: 5abcdd4
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Mon Dec 15 08:28:51 2014 -0800
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Mon Dec 15 08:28:51 2014 -0800
----------------------------------------------------------------------
.../jdbc/GenericJdbcFromInitializer.java | 5 +++--
.../jdbc/GenericJdbcToInitializer.java | 5 +++--
.../connector/kafka/KafkaToInitializer.java | 6 ++---
.../sqoop/connector/kite/KiteToInitializer.java | 8 +++----
.../idf/CSVIntermediateDataFormat.java | 4 ++--
.../connector/idf/IntermediateDataFormat.java | 10 ++++-----
.../org/apache/sqoop/driver/JobRequest.java | 10 ++++-----
.../org/apache/sqoop/driver/TestJobRequest.java | 23 +++++++++++---------
.../org/apache/sqoop/job/etl/Initializer.java | 8 +++----
9 files changed, 42 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java
index adf03d9..ff42949 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java
@@ -22,6 +22,7 @@ import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.LinkedList;
import java.util.List;
+import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
@@ -56,8 +57,8 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F
}
@Override
- public List<String> getJars(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
- List<String> jars = new LinkedList<String>();
+ public Set<String> getJars(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
+ Set<String> jars = super.getJars(context, linkConfig, fromJobConfig);
jars.add(ClassUtils.jarForClass(linkConfig.linkConfig.jdbcDriver));
return jars;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
index 930cfec..bc720da 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
@@ -22,6 +22,7 @@ import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.LinkedList;
import java.util.List;
+import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
@@ -53,8 +54,8 @@ public class GenericJdbcToInitializer extends Initializer<LinkConfiguration, ToJ
}
@Override
- public List<String> getJars(InitializerContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
- List<String> jars = new LinkedList<String>();
+ public Set<String> getJars(InitializerContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
+ Set<String> jars = super.getJars(context, linkConfig, toJobConfig);
jars.add(ClassUtils.jarForClass(linkConfig.linkConfig.jdbcDriver));
return jars;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/connector/connector-kafka/src/main/java/org/apache/sqoop/connector/kafka/KafkaToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-kafka/src/main/java/org/apache/sqoop/connector/kafka/KafkaToInitializer.java b/connector/connector-kafka/src/main/java/org/apache/sqoop/connector/kafka/KafkaToInitializer.java
index e1b065a..923d1aa 100644
--- a/connector/connector-kafka/src/main/java/org/apache/sqoop/connector/kafka/KafkaToInitializer.java
+++ b/connector/connector-kafka/src/main/java/org/apache/sqoop/connector/kafka/KafkaToInitializer.java
@@ -24,7 +24,7 @@ import org.apache.sqoop.job.etl.Initializer;
import org.apache.sqoop.job.etl.InitializerContext;
import org.apache.sqoop.utils.ClassUtils;
-import java.util.List;
+import java.util.Set;
public class KafkaToInitializer extends Initializer<LinkConfiguration,ToJobConfiguration> {
@@ -37,9 +37,9 @@ public class KafkaToInitializer extends Initializer<LinkConfiguration,ToJobConfi
@Override
- public List<String> getJars(InitializerContext context, LinkConfiguration
+ public Set<String> getJars(InitializerContext context, LinkConfiguration
linkConfiguration, ToJobConfiguration toJobConfiguration) {
- List<String> jars = super.getJars(context, linkConfiguration, toJobConfiguration);
+ Set<String> jars = super.getJars(context, linkConfiguration, toJobConfiguration);
// Jars for Kafka, Scala and Yammer (required by Kafka)
jars.add(ClassUtils.jarForClass("kafka.javaapi.producer.Producer"));
jars.add(ClassUtils.jarForClass("scala.collection.immutable.StringLike"));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
index 40b3abf..11233a8 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
@@ -28,7 +28,7 @@ import org.apache.sqoop.schema.NullSchema;
import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.utils.ClassUtils;
-import java.util.List;
+import java.util.Set;
/**
* This class allows connector to define initialization work for execution.
@@ -50,9 +50,9 @@ public class KiteToInitializer extends Initializer<LinkConfiguration,
}
@Override
- public List<String> getJars(InitializerContext context,
- LinkConfiguration linkConfig, ToJobConfiguration jobConfig) {
- List<String> jars = super.getJars(context, linkConfig, jobConfig);
+ public Set<String> getJars(InitializerContext context,
+ LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
+ Set<String> jars = super.getJars(context, linkConfig, toJobConfig);
jars.add(ClassUtils.jarForClass("org.kitesdk.data.Formats"));
jars.add(ClassUtils.jarForClass("com.fasterxml.jackson.databind.JsonNode"));
jars.add(ClassUtils.jarForClass("com.fasterxml.jackson.core.TreeNode"));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
index 73691ae..7529676 100644
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
@@ -372,9 +372,9 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
* {@inheritDoc}
*/
@Override
- public List<String> getJars() {
+ public Set<String> getJars() {
- List<String> jars = super.getJars();
+ Set<String> jars = super.getJars();
// Add JODA classes for IDF date/time handling
jars.add(ClassUtils.jarForClass(LocalDate.class));
jars.add(ClassUtils.jarForClass(LocalDateTime.class));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
index b8c8042..7ac44dc 100644
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
@@ -23,8 +23,8 @@ import org.apache.sqoop.schema.Schema;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
+import java.util.HashSet;
+import java.util.Set;
/**
* Abstract class representing a pluggable intermediate data format Sqoop
@@ -138,10 +138,10 @@ public abstract class IntermediateDataFormat<T> {
/**
* Provide the external jars that the IDF depends on
- * @return list of jars
+ * @return set of jars
*/
- public List<String> getJars() {
- return new LinkedList<String>();
+ public Set<String> getJars() {
+ return new HashSet<String>();
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/JobRequest.java b/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
index c9377a7..d2496bd 100644
--- a/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
+++ b/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
@@ -61,9 +61,9 @@ public class JobRequest {
SqoopConnector toConnector;
/**
- * List of required local jars for the job
+ * Set of required local jars for the job
*/
- List<String> jars;
+ Set<String> jars;
/**
* From entity
@@ -123,7 +123,7 @@ public class JobRequest {
Class<? extends IntermediateDataFormat<?>> toIDF;
public JobRequest() {
- this.jars = new LinkedList<String>();
+ this.jars = new HashSet<String>();
this.fromConnectorContext = new MutableMapContext();
this.toConnectorContext = new MutableMapContext();
this.driverContext = new MutableMapContext();
@@ -188,7 +188,7 @@ public class JobRequest {
}
}
- public List<String> getJars() {
+ public Set<String> getJars() {
return jars;
}
@@ -202,7 +202,7 @@ public class JobRequest {
addJar(ClassUtils.jarForClass(klass));
}
- public void addJars(List<String> jars) {
+ public void addJars(Set<String> jars) {
for(String j : jars) {
addJar(j);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java b/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java
index 9eff8a4..3f36030 100644
--- a/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java
+++ b/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java
@@ -18,6 +18,8 @@
package org.apache.sqoop.driver;
import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.sqoop.driver.JobRequest;
import org.apache.sqoop.utils.ClassUtils;
@@ -27,9 +29,6 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-/**
- *
- */
public class TestJobRequest {
private JobRequest jobRequest;
@@ -46,8 +45,8 @@ public class TestJobRequest {
jobRequest.addJar("A");
assertEquals(2, jobRequest.getJars().size());
- assertEquals("A", jobRequest.getJars().get(0));
- assertEquals("B", jobRequest.getJars().get(1));
+ assertTrue(jobRequest.getJars().contains("A"));
+ assertTrue(jobRequest.getJars().contains("B"));
}
@Test
@@ -59,12 +58,16 @@ public class TestJobRequest {
@Test
public void testAddJars() {
- jobRequest.addJars(Arrays.asList("A", "B"));
- jobRequest.addJars(Arrays.asList("B", "C"));
+ Set<String> set = new HashSet<String>();
+ set.addAll(Arrays.asList("A", "B"));
+ jobRequest.addJars(set);
+ set = new HashSet<String>();
+ set.addAll(Arrays.asList("B", "C"));
+ jobRequest.addJars(set);
assertEquals(3, jobRequest.getJars().size());
- assertEquals("A", jobRequest.getJars().get(0));
- assertEquals("B", jobRequest.getJars().get(1));
- assertEquals("C", jobRequest.getJars().get(2));
+ assertTrue(jobRequest.getJars().contains("A"));
+ assertTrue(jobRequest.getJars().contains("A"));
+ assertTrue(jobRequest.getJars().contains("C"));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java b/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
index 4dd6d5b..7dd156e 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
@@ -17,8 +17,8 @@
*/
package org.apache.sqoop.job.etl;
-import java.util.LinkedList;
-import java.util.List;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.sqoop.schema.NullSchema;
import org.apache.sqoop.schema.Schema;
@@ -53,9 +53,9 @@ public abstract class Initializer<LinkConfiguration, JobConfiguration> {
* In case of the TO initializer this will represent the TO job configuration
* @return
*/
- public List<String> getJars(InitializerContext context, LinkConfiguration linkConfiguration,
+ public Set<String> getJars(InitializerContext context, LinkConfiguration linkConfiguration,
JobConfiguration jobConfiguration) {
- return new LinkedList<String>();
+ return new HashSet<String>();
}
/**