You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pb...@apache.org on 2017/03/22 11:23:13 UTC
[08/50] [abbrv] oozie git commit: OOZIE-2709 Log the substituted pig
script by expanding macros (satishsaley)
OOZIE-2709 Log the substituted pig script by expanding macros (satishsaley)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/c58f5456
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/c58f5456
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/c58f5456
Branch: refs/heads/oya
Commit: c58f5456ddfdccce0c25a0ad80d86dba5b4ce944
Parents: 2a1ba19
Author: Satish Subhashrao Saley <sa...@yahoo-inc.com>
Authored: Thu Jan 19 10:12:39 2017 -0800
Committer: Satish Subhashrao Saley <sa...@yahoo-inc.com>
Committed: Thu Jan 19 10:12:39 2017 -0800
----------------------------------------------------------------------
core/src/main/resources/oozie-default.xml | 8 ++
release-log.txt | 1 +
sharelib/pig/pom.xml | 6 +-
.../org/apache/oozie/action/hadoop/PigMain.java | 82 +++++++++++++++++---
4 files changed, 84 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/c58f5456/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index 943f9bc..ad10386 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -1882,6 +1882,14 @@ will be the requeue interval for the actions which are waiting for a long time w
</property>
<property>
+ <name>oozie.action.pig.log.expandedscript</name>
+ <value>true</value>
+ <description>
+ Log the expanded pig script in launcher stdout log
+ </description>
+ </property>
+
+ <property>
<name>oozie.action.rootlogger.log.level</name>
<value>INFO</value>
<description>
http://git-wip-us.apache.org/repos/asf/oozie/blob/c58f5456/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 346a5ef..8a4aad0 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.4.0 release (trunk - unreleased)
+OOZIE-2709 Log the substituted pig script by expanding macros (satishsaley)
OOZIE-2732 Remove login server example (rkanter via abhishekbafna)
OOZIE-2756 Extend HTTPS configuration settings for embedded Jetty (asasvari via abhishekbafna)
OOZIE-2727 OozieDBCLI creates temporary directories and do not delete them (gezapeti via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/c58f5456/sharelib/pig/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/pig/pom.xml b/sharelib/pig/pom.xml
index 233873c..99148d7 100644
--- a/sharelib/pig/pom.xml
+++ b/sharelib/pig/pom.xml
@@ -140,7 +140,11 @@
</exclusion>
</exclusions>
</dependency>
-
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/oozie/blob/c58f5456/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
index 87c4cc4..7de2e45 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
@@ -20,8 +20,10 @@ package org.apache.oozie.action.hadoop;
import org.apache.pig.Main;
import org.apache.pig.PigRunner;
+import org.apache.pig.scripting.ScriptEngine;
import org.apache.pig.tools.pigstats.JobStats;
import org.apache.pig.tools.pigstats.PigStats;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -34,7 +36,9 @@ import java.io.FileOutputStream;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
@@ -48,6 +52,7 @@ import java.util.regex.Pattern;
public class PigMain extends LauncherMain {
private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<String>();
public static final int STRING_BUFFER_SIZE = 100;
+ public static final String LOG_EXPANDED_PIG_SCRIPT = LauncherMapper.ACTION_PREFIX + "pig.log.expandedscript";
private static final Pattern[] PIG_JOB_IDS_PATTERNS = {
Pattern.compile("HadoopJobId: (job_\\S*)"),
@@ -136,17 +141,7 @@ public class PigMain extends LauncherMain {
throw new RuntimeException("Error: Pig script file [" + script + "] does not exist");
}
- System.out.println("Pig script [" + script + "] content: ");
- System.out.println("------------------------");
- BufferedReader br = new BufferedReader(new FileReader(script));
- String line = br.readLine();
- while (line != null) {
- System.out.println(line);
- line = br.readLine();
- }
- br.close();
- System.out.println("------------------------");
- System.out.println();
+ printScript(script, "");
arguments.add("-file");
arguments.add(script);
@@ -212,6 +207,12 @@ public class PigMain extends LauncherMain {
arguments.add(pigArg);
}
+ if (actionConf.getBoolean(LOG_EXPANDED_PIG_SCRIPT, true)
+ // To avoid Pig running the embedded scripts on dryrun
+ && ScriptEngine.getSupportedScriptLang(script) == null) {
+ logExpandedScript(script, arguments);
+ }
+
System.out.println("Pig command arguments :");
for (String arg : arguments) {
System.out.println(" " + arg);
@@ -244,8 +245,65 @@ public class PigMain extends LauncherMain {
}
}
+ /**
+ * Logs the expanded
+ *
+ * @param script
+ * @param arguments
+ */
+ private void logExpandedScript(String script, List<String> arguments) {
+ List<String> dryrunArgs = new ArrayList<String>();
+ dryrunArgs.addAll(arguments);
+ dryrunArgs.add("-dryrun");
+ try {
+ PigRunner.run(dryrunArgs.toArray(new String[dryrunArgs.size()]), null);
+ printScript(script + ".expanded", "Expanded");
+ }
+ catch (Exception e) {
+ System.out.println("Failure while expanding pig script");
+ e.printStackTrace(System.out);
+ }
+ }
-
+ private void printScript(String name, String type) {
+ FileInputStream fin = null;
+ InputStreamReader inReader = null;
+ BufferedReader bufReader = null;
+ try {
+ File script = new File(name);
+ if (!script.exists()) {
+ return;
+ }
+ System.out.println("-----------------------------------------------------------");
+ System.out.println(type + " Pig script [" + name + "] content: ");
+ System.out.println("-----------------------------------------------------------");
+ fin = new FileInputStream(script);
+ inReader = new InputStreamReader(fin, "UTF-8");
+ bufReader = new BufferedReader(inReader);
+ String line = bufReader.readLine();
+ while (line != null) {
+ System.out.println(line);
+ line = bufReader.readLine();
+ }
+ bufReader.close();
+ }
+ catch (RuntimeException e) {
+ throw e;
+ }
+ catch (Exception e) {
+ System.out.println("Unable to read " + name);
+ e.printStackTrace(System.out);
+ }
+ finally {
+ IOUtils.closeQuietly(fin);
+ IOUtils.closeQuietly(inReader);
+ IOUtils.closeQuietly(bufReader);
+ }
+ System.out.println();
+ System.out.flush();
+ System.out.println("-----------------------------------------------------------");
+ System.out.println();
+ }
private void handleError(String pigLog) throws Exception {
System.err.println();
System.err.println("Pig logfile dump:");