You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/08/25 18:07:45 UTC
sqoop git commit: SQOOP-2516: Sqoop2: Findbugs: Fix warning in
execution module
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 b151ba384 -> 9f076edb0
SQOOP-2516: Sqoop2: Findbugs: Fix warning in execution module
(Colin Ma via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/9f076edb
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/9f076edb
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/9f076edb
Branch: refs/heads/sqoop2
Commit: 9f076edb0265fda24705320e944511dd0fee772e
Parents: b151ba3
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Tue Aug 25 09:07:19 2015 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Tue Aug 25 09:07:19 2015 -0700
----------------------------------------------------------------------
.../mapreduce/MapreduceExecutionEngine.java | 1 +
.../sqoop/job/mr/MRConfigurationUtils.java | 33 +++++++++++---------
.../apache/sqoop/job/mr/SqoopInputFormat.java | 3 +-
.../org/apache/sqoop/job/mr/SqoopMapper.java | 2 +-
.../sqoop/job/mr/SqoopNullOutputFormat.java | 2 +-
.../org/apache/sqoop/job/mr/SqoopSplit.java | 6 ++++
6 files changed, 29 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
index 3f79325..c8d210e 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
@@ -46,6 +46,7 @@ public class MapreduceExecutionEngine extends ExecutionEngine {
}
public void prepareJob(JobRequest jobRequest) {
+ assert jobRequest instanceof MRJobRequest;
MRJobRequest mrJobRequest = (MRJobRequest)jobRequest;
// Add jar dependencies
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
index e07ddd7..1e1b237 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
@@ -31,7 +31,9 @@ import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.utils.ClassUtils;
import org.json.simple.JSONObject;
+import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.Charset;
import java.util.Properties;
/**
@@ -89,12 +91,12 @@ public final class MRConfigurationUtils {
switch (type) {
case FROM:
job.getConfiguration().set(MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_LINK, obj.getClass().getName());
- job.getCredentials().addSecretKey(MR_JOB_CONFIG_FROM_CONNECTOR_LINK_KEY, ConfigUtils.toJson(obj).getBytes());
+ job.getCredentials().addSecretKey(MR_JOB_CONFIG_FROM_CONNECTOR_LINK_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8")));
break;
case TO:
job.getConfiguration().set(MR_JOB_CONFIG_CLASS_TO_CONNECTOR_LINK, obj.getClass().getName());
- job.getCredentials().addSecretKey(MR_JOB_CONFIG_TO_CONNECTOR_LINK_KEY, ConfigUtils.toJson(obj).getBytes());
+ job.getCredentials().addSecretKey(MR_JOB_CONFIG_TO_CONNECTOR_LINK_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8")));
break;
}
}
@@ -109,12 +111,12 @@ public final class MRConfigurationUtils {
switch (type) {
case FROM:
job.getConfiguration().set(MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_JOB, obj.getClass().getName());
- job.getCredentials().addSecretKey(MR_JOB_CONFIG_FROM_JOB_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes());
+ job.getCredentials().addSecretKey(MR_JOB_CONFIG_FROM_JOB_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8")));
break;
case TO:
job.getConfiguration().set(MR_JOB_CONFIG_CLASS_TO_CONNECTOR_JOB, obj.getClass().getName());
- job.getCredentials().addSecretKey(MR_JOB_CONFIG_TO_JOB_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes());
+ job.getCredentials().addSecretKey(MR_JOB_CONFIG_TO_JOB_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8")));
break;
}
}
@@ -128,7 +130,7 @@ public final class MRConfigurationUtils {
*/
public static void setDriverConfig(Job job, Object obj) {
job.getConfiguration().set(MR_JOB_CONFIG_DRIVER_CONFIG_CLASS, obj.getClass().getName());
- job.getCredentials().addSecretKey(MR_JOB_CONFIG_DRIVER_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes());
+ job.getCredentials().addSecretKey(MR_JOB_CONFIG_DRIVER_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8")));
}
/**
@@ -142,10 +144,10 @@ public final class MRConfigurationUtils {
String jsonSchema = SchemaSerialization.extractSchema(schema).toJSONString();
switch (type) {
case FROM:
- job.getCredentials().addSecretKey(SCHEMA_FROM_KEY,jsonSchema.getBytes());
+ job.getCredentials().addSecretKey(SCHEMA_FROM_KEY,jsonSchema.getBytes(Charset.forName("UTF-8")));
return;
case TO:
- job.getCredentials().addSecretKey(SCHEMA_TO_KEY, jsonSchema.getBytes());
+ job.getCredentials().addSecretKey(SCHEMA_TO_KEY, jsonSchema.getBytes(Charset.forName("UTF-8")));
return;
}
}
@@ -156,6 +158,7 @@ public final class MRConfigurationUtils {
* @return Configuration object
*/
public static Object getConnectorLinkConfig(Direction type, Configuration configuration) {
+ assert configuration instanceof JobConf;
switch (type) {
case FROM:
return loadConfiguration((JobConf) configuration, MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_LINK, MR_JOB_CONFIG_FROM_CONNECTOR_LINK_KEY);
@@ -174,6 +177,7 @@ public final class MRConfigurationUtils {
* @return Configuration object
*/
public static Object getConnectorJobConfig(Direction type, Configuration configuration) {
+ assert configuration instanceof JobConf;
switch (type) {
case FROM:
return loadConfiguration((JobConf) configuration, MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_JOB, MR_JOB_CONFIG_FROM_JOB_CONFIG_KEY);
@@ -192,6 +196,7 @@ public final class MRConfigurationUtils {
* @return Configuration object
*/
public static Object getDriverConfig(Configuration configuration) {
+ assert configuration instanceof JobConf;
return loadConfiguration((JobConf) configuration, MR_JOB_CONFIG_DRIVER_CONFIG_CLASS, MR_JOB_CONFIG_DRIVER_CONFIG_KEY);
}
@@ -204,6 +209,7 @@ public final class MRConfigurationUtils {
* @param configuration MapReduce configuration object
*/
public static Schema getConnectorSchema(Direction type, Configuration configuration) {
+ assert configuration instanceof JobConf;
switch (type) {
case FROM:
return getSchemaFromBytes(((JobConf) configuration).getCredentials().getSecretKey(SCHEMA_FROM_KEY));
@@ -228,7 +234,7 @@ public final class MRConfigurationUtils {
return null;
}
- JSONObject jsonSchema = JSONUtils.parse(new String(bytes));
+ JSONObject jsonSchema = JSONUtils.parse(new String(bytes, Charset.forName("UTF-8")));
return SchemaSerialization.restoreSchema(jsonSchema);
}
@@ -248,7 +254,7 @@ public final class MRConfigurationUtils {
return null;
}
- String json = new String(configuration.getCredentials().getSecretKey(valueProperty));
+ String json = new String(configuration.getCredentials().getSecretKey(valueProperty), Charset.forName("UTF-8"));
// Fill it with JSON data
ConfigUtils.fillValues(json, object);
@@ -262,15 +268,14 @@ public final class MRConfigurationUtils {
}
public static void configureLogging() {
- try {
+ try (InputStream resourceAsStream =
+ SqoopMapper.class.getResourceAsStream("/META-INF/log4j.properties")) {
Properties props = new Properties();
- InputStream resourceAsStream =
- SqoopMapper.class.getResourceAsStream("/META-INF/log4j.properties");
props.load(resourceAsStream);
PropertyConfigurator.configure(props);
- } catch (Exception e) {
+ } catch (RuntimeException | IOException e) {
System.err.println("Encountered exception while configuring logging " +
- "for sqoop: " + e);
+ "for sqoop: " + e);
}
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java
index 5aef878..67189a1 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java
@@ -75,8 +75,7 @@ public class SqoopInputFormat extends InputFormat<SqoopSplit, NullWritable> {
List<InputSplit> splits = new LinkedList<InputSplit>();
for (Partition partition : partitions) {
LOG.debug("Partition: " + partition);
- SqoopSplit split = new SqoopSplit();
- split.setPartition(partition);
+ SqoopSplit split = new SqoopSplit(partition);
splits.add(split);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
index 14fdfdc..937ef5a 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
@@ -149,7 +149,7 @@ public class SqoopMapper extends Mapper<SqoopSplit, NullWritable, SqoopWritable,
toIDF.setObjectData(matcher.getMatchingData(fromIDF.getObjectData()));
// NOTE: We do not use the reducer to do the writing (a.k.a LOAD in ETL). Hence the mapper sets up the writable
context.write(writable, NullWritable.get());
- } catch (Exception e) {
+ } catch (RuntimeException | IOException | InterruptedException e) {
throw new SqoopException(MRExecutionError.MAPRED_EXEC_0013, e);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java
index 6134106..88ab98e 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java
@@ -55,7 +55,7 @@ public class SqoopNullOutputFormat extends OutputFormat<SqoopWritable, NullWrita
return new SqoopDestroyerOutputCommitter();
}
- class SqoopDestroyerOutputCommitter extends OutputCommitter {
+ private static class SqoopDestroyerOutputCommitter extends OutputCommitter {
@Override
public void setupJob(JobContext jobContext) {
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java
index b81f4d6..bb6b2b0 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java
@@ -35,6 +35,12 @@ public class SqoopSplit extends InputSplit implements Writable {
private Partition partition;
+ public SqoopSplit() {}
+
+ public SqoopSplit(Partition partition) {
+ this.partition = partition;
+ }
+
public void setPartition(Partition partition) {
this.partition = partition;
}