You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2013/02/04 19:03:51 UTC
svn commit: r1442274 - in /oozie/branches/branch-3.3: ./
core/src/main/java/org/apache/oozie/action/hadoop/ webapp/src/main/webapp/
Author: rkanter
Date: Mon Feb 4 18:03:51 2013
New Revision: 1442274
URL: http://svn.apache.org/viewvc?rev=1442274&view=rev
Log:
OOZIE-1160 Oozie web-console to display all job URLs spawned by Pig (mona)
Modified:
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
oozie/branches/branch-3.3/release-log.txt
oozie/branches/branch-3.3/webapp/src/main/webapp/oozie-console.js
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java?rev=1442274&r1=1442273&r2=1442274&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java Mon Feb 4 18:03:51 2013
@@ -1037,6 +1037,7 @@ public class JavaActionExecutor extends
log.warn(errorReason);
}
context.setExecutionData(FAILED_KILLED, null);
+ setActionCompletionData(context, actionFs);
}
}
else {
@@ -1214,4 +1215,14 @@ public class JavaActionExecutor extends
protected String getDefaultShareLibName(Element actionXml) {
return null;
}
+
+ /**
+ * Sets some data for the action on completion
+ *
+ * @param context executor context
+ * @param actionFs the FileSystem object
+ */
+ protected void setActionCompletionData(Context context, FileSystem actionFs) throws IOException,
+ HadoopAccessorException, URISyntaxException {
+ }
}
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java?rev=1442274&r1=1442273&r2=1442274&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java Mon Feb 4 18:03:51 2013
@@ -519,13 +519,13 @@ public class LauncherMapper<K1, V1, K2,
System.out.println();
System.out.println("<<< Invocation of Main class completed <<<");
System.out.println();
+ handleExternalChildIDs(reporter);
}
if (errorMessage == null) {
File outputData = new File(System.getProperty("oozie.action.output.properties"));
- FileSystem fs = null;
if (outputData.exists()) {
URI actionDirUri = new Path(actionDir, ACTION_OUTPUT_PROPS).toUri();
- fs = FileSystem.get(actionDirUri, getJobConf());
+ FileSystem fs = FileSystem.get(actionDirUri, getJobConf());
fs.copyFromLocalFile(new Path(outputData.toString()), new Path(actionDir,
ACTION_OUTPUT_PROPS));
reporter.incrCounter(COUNTER_GROUP, COUNTER_OUTPUT_DATA, 1);
@@ -546,8 +546,8 @@ public class LauncherMapper<K1, V1, K2,
System.out.println("=======================");
System.out.println();
}
- handleActionStatsData(fs, reporter);
- handleExternalChildIDs(fs, reporter);
+ handleActionStatsData(reporter);
+ handleExternalChildIDs(reporter);
File newId = new File(System.getProperty("oozie.action.newId.properties"));
if (newId.exists()) {
Properties props = new Properties();
@@ -556,7 +556,7 @@ public class LauncherMapper<K1, V1, K2,
throw new IllegalStateException("ID swap file does not have [id] property");
}
URI actionDirUri = new Path(actionDir, ACTION_NEW_ID_PROPS).toUri();
- fs = FileSystem.get(actionDirUri, getJobConf());
+ FileSystem fs = FileSystem.get(actionDirUri, getJobConf());
fs.copyFromLocalFile(new Path(newId.toString()), new Path(actionDir, ACTION_NEW_ID_PROPS));
reporter.incrCounter(COUNTER_GROUP, COUNTER_DO_ID_SWAP, 1);
@@ -611,7 +611,7 @@ public class LauncherMapper<K1, V1, K2,
return jobConf;
}
- private void handleActionStatsData(FileSystem fs, Reporter reporter) throws IOException, LauncherException {
+ private void handleActionStatsData(Reporter reporter) throws IOException, LauncherException {
File actionStatsData = new File(System.getProperty(EXTERNAL_ACTION_STATS));
// If stats are stored by the action, then stats file should exist
if (actionStatsData.exists()) {
@@ -626,19 +626,19 @@ public class LauncherMapper<K1, V1, K2,
}
// copy the stats file to hdfs path which can be accessed by Oozie server
URI actionDirUri = new Path(actionDir, ACTION_STATS_PROPS).toUri();
- fs = FileSystem.get(actionDirUri, getJobConf());
+ FileSystem fs = FileSystem.get(actionDirUri, getJobConf());
fs.copyFromLocalFile(new Path(actionStatsData.toString()), new Path(actionDir,
ACTION_STATS_PROPS));
}
}
- private void handleExternalChildIDs(FileSystem fs, Reporter reporter) throws IOException {
+ private void handleExternalChildIDs(Reporter reporter) throws IOException {
File externalChildIDs = new File(System.getProperty(EXTERNAL_CHILD_IDS));
// if external ChildIDs are stored by the action, then the file should exist
if (externalChildIDs.exists()) {
// copy the externalChildIDs file to hdfs path which can be accessed by Oozie server
URI actionDirUri = new Path(actionDir, ACTION_EXTERNAL_CHILD_IDS_PROPS).toUri();
- fs = FileSystem.get(actionDirUri, getJobConf());
+ FileSystem fs = FileSystem.get(actionDirUri, getJobConf());
fs.copyFromLocalFile(new Path(externalChildIDs.toString()), new Path(actionDir,
ACTION_EXTERNAL_CHILD_IDS_PROPS));
}
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java?rev=1442274&r1=1442273&r2=1442274&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java Mon Feb 4 18:03:51 2013
@@ -17,14 +17,10 @@
*/
package org.apache.oozie.action.hadoop;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Map;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.Counters.Counter;
-import org.apache.oozie.util.ParamChecker;
import org.apache.pig.tools.pigstats.JobStats;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.PigStatsUtil;
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java?rev=1442274&r1=1442273&r2=1442274&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java Mon Feb 4 18:03:51 2013
@@ -170,6 +170,16 @@ public class PigActionExecutor extends J
return stats;
}
+ @Override
+ protected void setActionCompletionData(Context context, FileSystem fs) throws HadoopAccessorException, IOException,
+ URISyntaxException {
+ String data = getExternalChildIDs(context, fs);
+ if (data != null && !data.isEmpty()) {
+ context.setExternalChildIDs(data);
+ XLog.getLog(getClass()).info(XLog.STD, "Hadoop Jobs launched : [{0}]", data);
+ }
+ }
+
private String getExternalChildIDs(Context context, FileSystem actionFs) throws IOException,
HadoopAccessorException, URISyntaxException {
Path actionOutput = LauncherMapper.getExternalChildIDsDataPath(context.getActionDir());
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java?rev=1442274&r1=1442273&r2=1442274&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java Mon Feb 4 18:03:51 2013
@@ -281,6 +281,12 @@ public class PigMain extends LauncherMai
if (pigRunnerExists) {
System.out.println("Run pig script using PigRunner.run() for Pig version 0.8+");
PigStats stats = PigRunner.run(args, null);
+ String jobIds = getHadoopJobIds(stats);
+ if (jobIds != null && !jobIds.isEmpty()) {
+ System.out.println("Hadoop Job IDs executed by Pig: " + jobIds);
+ File f = new File(System.getProperty(EXTERNAL_CHILD_IDS));
+ writeExternalData(jobIds, f);
+ }
// isSuccessful is the API from 0.9 supported by both PigStats and
// EmbeddedPigStats
if (!stats.isSuccessful()) {
@@ -295,12 +301,6 @@ public class PigMain extends LauncherMai
if (resetSecurityManager) {
return;
}
- String jobIds = getHadoopJobIds(stats);
- if (jobIds != null) {
- System.out.println(" Hadoop Job IDs executed by Pig: " + jobIds);
- File f = new File(System.getProperty(EXTERNAL_CHILD_IDS));
- writeExternalData(jobIds, f);
- }
// Retrieve stats only if user has specified in workflow
// configuration
if (retrieveStats) {
Modified: oozie/branches/branch-3.3/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/release-log.txt?rev=1442274&r1=1442273&r2=1442274&view=diff
==============================================================================
--- oozie/branches/branch-3.3/release-log.txt (original)
+++ oozie/branches/branch-3.3/release-log.txt Mon Feb 4 18:03:51 2013
@@ -1,5 +1,6 @@
-- Oozie 3.3.2 (unreleased)
+OOZIE-1160 Oozie web-console to display all job URLs spawned by Pig (mona)
OOZIE-1177 HostnameFilter should only catch UnknownHostException, not Exception (rkanter)
OOZIE-945 BundleSubmitXCommand.submit() doesn't properly remove comments (jaoki via rkanter)
OOZIE-1171 HostnameFilter should handle hostname resolution failures and continue processing (tucu via rkanter)
Modified: oozie/branches/branch-3.3/webapp/src/main/webapp/oozie-console.js
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/webapp/src/main/webapp/oozie-console.js?rev=1442274&r1=1442273&r2=1442274&view=diff
==============================================================================
--- oozie/branches/branch-3.3/webapp/src/main/webapp/oozie-console.js (original)
+++ oozie/branches/branch-3.3/webapp/src/main/webapp/oozie-console.js Mon Feb 4 18:03:51 2013
@@ -246,7 +246,7 @@ function jobDetailsPopup(response, reque
var appName = jobDetails["appName"];
var jobActionStatus = new Ext.data.JsonStore({
data: jobDetails["actions"],
- fields: ['id', 'name', 'type', 'startTime', 'retries', 'consoleUrl', 'endTime', 'externalId', 'status', 'trackerUri', 'workflowId', 'errorCode', 'errorMessage', 'conf', 'transition', 'externalStatus']
+ fields: ['id', 'name', 'type', 'startTime', 'retries', 'consoleUrl', 'endTime', 'externalId', 'status', 'trackerUri', 'workflowId', 'errorCode', 'errorMessage', 'conf', 'transition', 'externalStatus', 'externalChildIDs']
});
var formFieldSet = new Ext.form.FieldSet({
@@ -492,8 +492,8 @@ function jobDetailsPopup(response, reque
width: 400,
value: actionStatus["trackerUri"]
- }, ]
- });
+ }
+ ]});
var detail = new Ext.FormPanel({
frame: true,
labelAlign: 'right',
@@ -501,6 +501,8 @@ function jobDetailsPopup(response, reque
width: 540,
items: [formFieldSet]
});
+ var urlUnit = new Ext.FormPanel();
+ populateUrlUnit(actionStatus, urlUnit);
var win = new Ext.Window({
title: 'Action (Name: ' + actionStatus["name"] + '/JobId: ' + workflowId + ')',
closable: true,
@@ -525,13 +527,76 @@ function jobDetailsPopup(response, reque
autoScroll: true,
value: actionStatus["conf"]
})
- }, ]
+ }, {
+ title: 'Child Job URLs',
+ autoScroll: true,
+ frame: true,
+ labelAlign: 'right',
+ labelWidth: 70,
+ items: urlUnit
+ }],
+ tbar: [{
+ text: " ",
+ icon: 'ext-2.2/resources/images/default/grid/refresh.gif',
+ handler: function() {
+ refreshActionDetails(workflowId+"@"+actionStatus["name"], detail, urlUnit);
+ }
+ }]
})]
});
win.setPosition(50, 50);
win.show();
}
}
+
+ function populateUrlUnit(actionStatus, urlUnit) {
+ var consoleUrl = actionStatus["consoleUrl"];
+ var externalChildIDs = actionStatus["externalChildIDs"];
+ if(undefined !== consoleUrl && null !== consoleUrl && undefined !== externalChildIDs && null !== externalChildIDs) {
+ var urlPrefix = consoleUrl.trim().split(/_/)[0];
+ //externalChildIds is a comma-separated string of each child job ID.
+ //Create URL list by appending jobID portion after stripping "job"
+ var jobIds = externalChildIDs.split(/,/);
+ var count = 1;
+ jobIds.forEach(function(jobId) {
+ jobId = jobId.trim().split(/job/)[1];
+ var jobUrl = new Ext.form.TriggerField({
+ fieldLabel: 'Child Job ' + count,
+ editable: false,
+ name: 'childJobURLs',
+ width: 400,
+ value: urlPrefix + jobId,
+ triggerClass: 'x-form-search-trigger',
+ onTriggerClick: function() {
+ window.open(urlPrefix + jobId);
+ }
+ });
+ if(jobId != undefined) {
+ urlUnit.add(jobUrl);
+ count++;
+ }
+ });
+ } else {
+ var note = new Ext.form.TextField({
+ fieldLabel: 'Child Job',
+ value: 'n/a'
+ });
+ urlUnit.add(note);
+ }
+ }
+
+ function refreshActionDetails(actionId, detail, urlUnit) {
+ Ext.Ajax.request({
+ url: getOozieBase() + 'job/' + actionId + "?timezone=" + getTimeZone(),
+ success: function(response, request) {
+ var results = eval("(" + response.responseText + ")");
+ detail.getForm().setValues(results);
+ urlUnit.getForm().setValues(results);
+ populateUrlUnit(results, urlUnit);
+ }
+ });
+ }
+
var dagImg = new Ext.ux.Image({
id: 'dagImage',
url: getOozieBase() + 'job/' + workflowId + "?show=graph",
@@ -701,7 +766,7 @@ function coordJobDetailsPopup(response,
var appName = jobDetails["coordJobName"];
var jobActionStatus = new Ext.data.JsonStore({
data: jobDetails["actions"],
- fields: ['id', 'name', 'type', 'createdConf', 'runConf', 'actionNumber', 'createdTime', 'externalId', 'lastModifiedTime', 'nominalTime', 'status', 'missingDependencies', 'externalStatus', 'trackerUri', 'consoleUrl', 'errorCode', 'errorMessage', 'actions']
+ fields: ['id', 'name', 'type', 'createdConf', 'runConf', 'actionNumber', 'createdTime', 'externalId', 'lastModifiedTime', 'nominalTime', 'status', 'missingDependencies', 'externalStatus', 'trackerUri', 'consoleUrl', 'errorCode', 'errorMessage', 'actions', 'externalChildIDs']
});