You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ge...@apache.org on 2017/06/27 09:58:38 UTC
oozie git commit: OOZIE-2943 Fix Findbugs warnings in
oozie-sharelib-pig (Jan Hentschel via gezapeti)
Repository: oozie
Updated Branches:
refs/heads/master 9937e4c4a -> 678f1b2b2
OOZIE-2943 Fix Findbugs warnings in oozie-sharelib-pig (Jan Hentschel via gezapeti)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/678f1b2b
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/678f1b2b
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/678f1b2b
Branch: refs/heads/master
Commit: 678f1b2b2cc9a2456c7e3d9f693d2637f1f03257
Parents: 9937e4c
Author: Gezapeti Cseh <ge...@gmail.com>
Authored: Tue Jun 27 11:58:22 2017 +0200
Committer: Gezapeti Cseh <ge...@gmail.com>
Committed: Tue Jun 27 11:58:22 2017 +0200
----------------------------------------------------------------------
release-log.txt | 1 +
.../oozie/action/hadoop/OoziePigStats.java | 8 +-
.../org/apache/oozie/action/hadoop/PigMain.java | 33 ++++-----
.../oozie/action/hadoop/PigMainWithOldAPI.java | 78 +++++++++++---------
4 files changed, 61 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/678f1b2b/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index f862b39..97884b6 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.0.0 release (trunk - unreleased)
+OOZIE-2943 Fix Findbugs warnings in oozie-sharelib-pig (Jan Hentschel via gezapeti)
OOZIE-2846 TestSSLServerConnectorFactory fails with Jetty 9.3 and 9.4 (gezapeti)
OOZIE-2936 Fix Javadoc warnings from build (Jan Hentschel via gezapeti)
OOZIE-2920 Document Distcp can copy files within a cluster (Artem Ervits via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/678f1b2b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java
index c3a31b5..b2fcfc4 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java
@@ -119,18 +119,16 @@ public class OoziePigStats extends ActionStats {
jobStatsGroup.put("MULTI_STORE_COUNTERS", toJSONFromMultiStoreCounters(jobStats.getMultiStoreCounters()));
return jobStatsGroup;
-
}
// multistorecounters to JSON
@SuppressWarnings("unchecked")
private static JSONObject toJSONFromMultiStoreCounters(Map<String, Long> map) {
JSONObject group = new JSONObject();
- for (String cName : map.keySet()) {
- group.put(cName, map.get(cName));
+ for (Map.Entry cName : map.entrySet()) {
+ group.put(cName.getKey(), cName.getValue());
}
return group;
-
}
// hadoop counters to JSON
@@ -151,7 +149,5 @@ public class OoziePigStats extends ActionStats {
groups.put(gName, group);
}
return groups;
-
}
-
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/678f1b2b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
index 5a9123a..b6b30eb 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
@@ -33,15 +33,14 @@ import org.apache.hadoop.fs.Path;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
-import java.io.FileWriter;
import java.io.OutputStream;
import java.io.FileOutputStream;
import java.io.BufferedReader;
-import java.io.FileReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
@@ -52,7 +51,7 @@ import java.util.Set;
import java.util.regex.Pattern;
public class PigMain extends LauncherMain {
- private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<String>();
+ private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<>();
public static final int STRING_BUFFER_SIZE = 100;
public static final String LOG_EXPANDED_PIG_SCRIPT = LauncherMapper.ACTION_PREFIX + "pig.log.expandedscript";
@@ -127,14 +126,14 @@ public class PigMain extends LauncherMain {
String callerId = "oozie:" + System.getProperty("oozie.job.id");
pigProperties.setProperty("pig.log.trace.id", callerId);
- OutputStream os = new FileOutputStream("pig.properties");
- pigProperties.store(os, "");
- os.close();
+ try (OutputStream os = new FileOutputStream("pig.properties")) {
+ pigProperties.store(os, "");
+ }
logMasking("pig.properties:", Arrays.asList("password"),
(Iterable<Map.Entry<String, String>>)(Iterable<?>) pigProperties.entrySet());
- List<String> arguments = new ArrayList<String>();
+ List<String> arguments = new ArrayList<>();
String script = actionConf.get(PigActionExecutor.PIG_SCRIPT);
if (script == null) {
@@ -247,7 +246,7 @@ public class PigMain extends LauncherMain {
* @param arguments
*/
private void logExpandedScript(String script, List<String> arguments) {
- List<String> dryrunArgs = new ArrayList<String>();
+ List<String> dryrunArgs = new ArrayList<>();
dryrunArgs.addAll(arguments);
dryrunArgs.add("-dryrun");
try {
@@ -304,13 +303,13 @@ public class PigMain extends LauncherMain {
System.err.println("Pig logfile dump:");
System.err.println();
try {
- BufferedReader reader = new BufferedReader(new FileReader(pigLog));
- String line = reader.readLine();
- while (line != null) {
- System.err.println(line);
- line = reader.readLine();
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(pigLog), "UTF-8"))) {
+ String line = reader.readLine();
+ while (line != null) {
+ System.err.println(line);
+ line = reader.readLine();
+ }
}
- reader.close();
}
catch (FileNotFoundException e) {
System.err.println("pig log file: " + pigLog + " not found.");
@@ -330,9 +329,9 @@ public class PigMain extends LauncherMain {
protected void runPigJob(String[] args, String pigLog, boolean resetSecurityManager, boolean retrieveStats) throws Exception {
// running as from the command line
boolean pigRunnerExists = true;
- Class klass;
+
try {
- klass = Class.forName("org.apache.pig.PigRunner");
+ Class.forName("org.apache.pig.PigRunner");
}
catch (ClassNotFoundException ex) {
pigRunnerExists = false;
@@ -405,7 +404,7 @@ public class PigMain extends LauncherMain {
private static void writeExternalData(String data, File f) throws IOException {
BufferedWriter out = null;
try {
- out = new BufferedWriter(new FileWriter(f));
+ out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f), "UTF-8"));
out.write(data);
}
finally {
http://git-wip-us.apache.org/repos/asf/oozie/blob/678f1b2b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
index 0ee4b0b..df31655 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
@@ -22,11 +22,12 @@ import org.apache.pig.Main;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import java.io.FileInputStream;
import java.io.FileNotFoundException;
+import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.FileOutputStream;
import java.io.BufferedReader;
-import java.io.FileReader;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
@@ -37,7 +38,7 @@ import java.util.Properties;
import java.util.Set;
public class PigMainWithOldAPI extends LauncherMain {
- private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<String>();
+ private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<>();
static {
DISALLOWED_PIG_OPTIONS.add("-4");
@@ -98,9 +99,9 @@ public class PigMainWithOldAPI extends LauncherMain {
System.out.println("Non-kerberos execution");
}
- OutputStream os = new FileOutputStream("pig.properties");
- pigProperties.store(os, "");
- os.close();
+ try (OutputStream os = new FileOutputStream("pig.properties")) {
+ pigProperties.store(os, "");
+ }
System.out.println();
System.out.println("pig.properties content:");
@@ -110,7 +111,7 @@ public class PigMainWithOldAPI extends LauncherMain {
System.out.println("------------------------");
System.out.println();
- List<String> arguments = new ArrayList<String>();
+ List<String> arguments = new ArrayList<>();
String script = actionConf.get("oozie.pig.script");
if (script == null) {
@@ -123,13 +124,16 @@ public class PigMainWithOldAPI extends LauncherMain {
System.out.println("Pig script [" + script + "] content: ");
System.out.println("------------------------");
- BufferedReader br = new BufferedReader(new FileReader(script));
- String line = br.readLine();
- while (line != null) {
- System.out.println(line);
+
+ String line;
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(script), "UTF-8"))) {
line = br.readLine();
+ while (line != null) {
+ System.out.println(line);
+ line = br.readLine();
+ }
}
- br.close();
+
System.out.println("------------------------");
System.out.println();
@@ -216,13 +220,14 @@ public class PigMainWithOldAPI extends LauncherMain {
System.err.println("Pig logfile dump:");
System.err.println();
try {
- BufferedReader reader = new BufferedReader(new FileReader(pigLog));
- line = reader.readLine();
- while (line != null) {
- System.err.println(line);
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(pigLog),
+ "UTF-8"))) {
line = reader.readLine();
+ while (line != null) {
+ System.err.println(line);
+ line = reader.readLine();
+ }
}
- reader.close();
}
catch (FileNotFoundException e) {
System.err.println("pig log file: " + pigLog + " not found.");
@@ -239,9 +244,11 @@ public class PigMainWithOldAPI extends LauncherMain {
// harvesting and recording Hadoop Job IDs
Properties jobIds = getHadoopJobIds(logFile);
File file = new File(System.getProperty(OUTPUT_PROPERTIES));
- os = new FileOutputStream(file);
- jobIds.store(os, "");
- os.close();
+
+ try (OutputStream os = new FileOutputStream(file)) {
+ jobIds.store(os, "");
+ }
+
System.out.println(" Hadoop Job IDs executed by Pig: " + jobIds.getProperty(HADOOP_JOBS));
System.out.println();
}
@@ -254,34 +261,33 @@ public class PigMainWithOldAPI extends LauncherMain {
private static final String JOB_ID_LOG_PREFIX = "HadoopJobId: ";
protected Properties getHadoopJobIds(String logFile) throws IOException {
- int jobCount = 0;
Properties props = new Properties();
StringBuffer sb = new StringBuffer(100);
- if (new File(logFile).exists() == false) {
+ if (!new File(logFile).exists()) {
System.err.println("pig log file: " + logFile + " not present. Therefore no Hadoop jobids found");
props.setProperty(HADOOP_JOBS, "");
}
else {
- BufferedReader br = new BufferedReader(new FileReader(logFile));
- String line = br.readLine();
- String separator = "";
- while (line != null) {
- if (line.contains(JOB_ID_LOG_PREFIX)) {
- int jobIdStarts = line.indexOf(JOB_ID_LOG_PREFIX) + JOB_ID_LOG_PREFIX.length();
- String jobId = line.substring(jobIdStarts);
- int jobIdEnds = jobId.indexOf(" ");
- if (jobIdEnds > -1) {
- jobId = jobId.substring(0, jobId.indexOf(" "));
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(logFile), "UTF-8"))) {
+ String line = br.readLine();
+ String separator = "";
+ while (line != null) {
+ if (line.contains(JOB_ID_LOG_PREFIX)) {
+ int jobIdStarts = line.indexOf(JOB_ID_LOG_PREFIX) + JOB_ID_LOG_PREFIX.length();
+ String jobId = line.substring(jobIdStarts);
+ int jobIdEnds = jobId.indexOf(" ");
+ if (jobIdEnds > -1) {
+ jobId = jobId.substring(0, jobId.indexOf(" "));
+ }
+ sb.append(separator).append(jobId);
+ separator = ",";
}
- sb.append(separator).append(jobId);
- separator = ",";
+ line = br.readLine();
}
- line = br.readLine();
}
- br.close();
+
props.setProperty(HADOOP_JOBS, sb.toString());
}
return props;
}
-
}