You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by mo...@apache.org on 2013/08/28 20:48:13 UTC
svn commit: r1518326 [2/2] - in /oozie/trunk:
core/src/main/java/org/apache/oozie/action/hadoop/
core/src/test/java/org/apache/oozie/action/hadoop/
core/src/test/java/org/apache/oozie/command/wf/
core/src/test/java/org/apache/oozie/service/ sharelib/hi...
Modified: oozie/trunk/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java?rev=1518326&r1=1518325&r2=1518326&view=diff
==============================================================================
--- oozie/trunk/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java (original)
+++ oozie/trunk/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java Wed Aug 28 18:48:12 2013
@@ -27,7 +27,6 @@ import org.apache.hadoop.mapred.RunningJ
import org.apache.hadoop.mapred.JobID;
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
-import org.apache.oozie.action.hadoop.ActionExecutorTestCase.Context;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.service.WorkflowAppService;
@@ -226,14 +225,16 @@ public class TestPigActionExecutor exten
assertTrue(launcherJob.isSuccessful());
sleep(2000);
- assertFalse(LauncherMapperHelper.hasIdSwap(launcherJob));
- if (checkForSuccess) {
- assertTrue(LauncherMapperHelper.hasStatsData(launcherJob));
- }
PigActionExecutor ae = new PigActionExecutor();
ae.check(context, context.getAction());
ae.end(context, context.getAction());
+
+ if (checkForSuccess) {
+ assertFalse(context.getExternalChildIDs().equals(launcherId));
+ assertNotNull(context.getAction().getStats());
+ }
+
assertTrue(launcherId.equals(context.getAction().getExternalId()));
if (checkForSuccess) {
assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
@@ -262,7 +263,10 @@ public class TestPigActionExecutor exten
final RunningJob launcherJob = submitAction(context);
evaluateLauncherJob(launcherJob);
assertTrue(launcherJob.isSuccessful());
- assertTrue(LauncherMapperHelper.hasStatsData(launcherJob));
+
+ Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
+ new XConfiguration());
+ assertTrue(LauncherMapperHelper.hasStatsData(actionData));
PigActionExecutor ae = new PigActionExecutor();
WorkflowAction wfAction = context.getAction();
@@ -349,7 +353,10 @@ public class TestPigActionExecutor exten
final RunningJob launcherJob = submitAction(context);
evaluateLauncherJob(launcherJob);
assertTrue(launcherJob.isSuccessful());
- assertFalse(LauncherMapperHelper.hasStatsData(launcherJob));
+
+ Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
+ new XConfiguration());
+ assertFalse(LauncherMapperHelper.hasStatsData(actionData));
PigActionExecutor ae = new PigActionExecutor();
WorkflowAction wfAction = context.getAction();
Modified: oozie/trunk/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java?rev=1518326&r1=1518325&r2=1518326&view=diff
==============================================================================
--- oozie/trunk/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java (original)
+++ oozie/trunk/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java Wed Aug 28 18:48:12 2013
@@ -93,12 +93,12 @@ public class TestPigMain extends PigTest
File statsDataFile = new File(getTestCaseDir(), "statsdata.properties");
- File hadoopIdsFile = new File(getTestCaseDir(), "hadoopIds.properties");
+ File hadoopIdsFile = new File(getTestCaseDir(), "hadoopIds");
setSystemProperty("oozie.launcher.job.id", "" + System.currentTimeMillis());
setSystemProperty("oozie.action.conf.xml", actionXml.getAbsolutePath());
setSystemProperty("oozie.action.stats.properties", statsDataFile.getAbsolutePath());
- setSystemProperty("oozie.action.externalChildIDs.properties", hadoopIdsFile.getAbsolutePath());
+ setSystemProperty("oozie.action.externalChildIDs", hadoopIdsFile.getAbsolutePath());
URL url = Thread.currentThread().getContextClassLoader().getResource("PigMain.txt");
Modified: oozie/trunk/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java?rev=1518326&r1=1518325&r2=1518326&view=diff
==============================================================================
--- oozie/trunk/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java (original)
+++ oozie/trunk/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java Wed Aug 28 18:48:12 2013
@@ -186,14 +186,11 @@ public class SqoopMain extends LauncherM
// harvesting and recording Hadoop Job IDs
Properties jobIds = getHadoopJobIds(logFile, SQOOP_JOB_IDS_PATTERNS);
- File file = new File(System.getProperty("oozie.action.output.properties"));
+ File file = new File(System.getProperty(LauncherMapper.ACTION_PREFIX
+ + LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS));
OutputStream os = new FileOutputStream(file);
- try {
- jobIds.store(os, "");
- }
- finally {
- os.close();
- }
+ os.write(jobIds.getProperty(HADOOP_JOBS).getBytes());
+ os.close();
System.out.println(" Hadoop Job IDs executed by Sqoop: " + jobIds.getProperty(HADOOP_JOBS));
System.out.println();
}
Modified: oozie/trunk/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java?rev=1518326&r1=1518325&r2=1518326&view=diff
==============================================================================
--- oozie/trunk/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java (original)
+++ oozie/trunk/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java Wed Aug 28 18:48:12 2013
@@ -50,7 +50,7 @@ import java.sql.Statement;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.List;
-import java.util.Properties;
+import java.util.Map;
public class TestSqoopActionExecutor extends ActionExecutorTestCase {
@@ -225,14 +225,14 @@ public class TestSqoopActionExecutor ext
}
});
assertTrue(launcherJob.isSuccessful());
-
- assertFalse(LauncherMapperHelper.hasIdSwap(launcherJob));
+ Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
+ new XConfiguration());
+ assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
SqoopActionExecutor ae = new SqoopActionExecutor();
ae.check(context, context.getAction());
assertTrue(launcherId.equals(context.getAction().getExternalId()));
assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
- assertNotNull(context.getAction().getData());
assertNotNull(context.getAction().getExternalChildIDs());
ae.end(context, context.getAction());
assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
@@ -253,11 +253,7 @@ public class TestSqoopActionExecutor ext
br.close();
assertEquals(3, count);
- assertNotNull(context.getAction().getData());
- Properties outputData = new Properties();
- outputData.load(new StringReader(context.getAction().getData()));
- assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS));
- assertTrue(outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length() > 0);
+ assertTrue(actionData.get(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS).trim().length() > 0);
}
public void testSqoopEval() throws Exception {
@@ -272,14 +268,14 @@ public class TestSqoopActionExecutor ext
}
});
assertTrue(launcherJob.isSuccessful());
-
- assertFalse(LauncherMapperHelper.hasIdSwap(launcherJob));
+ Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
+ new XConfiguration());
+ assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
SqoopActionExecutor ae = new SqoopActionExecutor();
ae.check(context, context.getAction());
assertTrue(launcherId.equals(context.getAction().getExternalId()));
assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
- assertNotNull(context.getAction().getData());
assertNotNull(context.getAction().getExternalChildIDs());
assertEquals(0, context.getAction().getExternalChildIDs().length());
ae.end(context, context.getAction());
@@ -289,11 +285,7 @@ public class TestSqoopActionExecutor ext
assertNotNull(hadoopCounters);
assertTrue(hadoopCounters.isEmpty());
- assertNotNull(context.getAction().getData());
- Properties outputData = new Properties();
- outputData.load(new StringReader(context.getAction().getData()));
- assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS));
- assertEquals(0, outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length());
+ assertTrue(actionData.get(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS).isEmpty());
}
public void testSqoopActionFreeFormQuery() throws Exception {
@@ -308,14 +300,14 @@ public class TestSqoopActionExecutor ext
}
});
assertTrue(launcherJob.isSuccessful());
-
- assertFalse(LauncherMapperHelper.hasIdSwap(launcherJob));
+ Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
+ new XConfiguration());
+ assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
SqoopActionExecutor ae = new SqoopActionExecutor();
ae.check(context, context.getAction());
assertTrue(launcherId.equals(context.getAction().getExternalId()));
assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
- assertNotNull(context.getAction().getData());
assertNotNull(context.getAction().getExternalChildIDs());
ae.end(context, context.getAction());
assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
@@ -344,11 +336,7 @@ public class TestSqoopActionExecutor ext
}
assertEquals(3, count);
- assertNotNull(context.getAction().getData());
- Properties outputData = new Properties();
- outputData.load(new StringReader(context.getAction().getData()));
- assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS));
- assertTrue(outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length() > 0);
+ assertTrue(actionData.get(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS).trim().length() > 0);
}
Modified: oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java?rev=1518326&r1=1518325&r2=1518326&view=diff
==============================================================================
--- oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java (original)
+++ oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java Wed Aug 28 18:48:12 2013
@@ -50,6 +50,7 @@ import java.io.StringReader;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Scanner;
import java.util.jar.JarOutputStream;
import java.util.regex.Pattern;
@@ -343,8 +344,9 @@ public class TestMapReduceActionExecutor
}
});
assertTrue(launcherJob.isSuccessful());
-
- assertTrue(LauncherMapperHelper.hasIdSwap(launcherJob));
+ Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
+ new XConfiguration());
+ assertTrue(LauncherMapperHelper.hasIdSwap(actionData));
MapReduceActionExecutor ae = new MapReduceActionExecutor();
ae.check(context, context.getAction());
@@ -393,8 +395,9 @@ public class TestMapReduceActionExecutor
}
});
assertTrue(launcherJob.isSuccessful());
-
- assertTrue(LauncherMapperHelper.hasIdSwap(launcherJob));
+ Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
+ new XConfiguration());
+ assertTrue(LauncherMapperHelper.hasIdSwap(actionData));
MapReduceActionExecutor ae = new MapReduceActionExecutor();
ae.check(context, context.getAction());
@@ -716,8 +719,9 @@ public class TestMapReduceActionExecutor
}
});
assertTrue(launcherJob.isSuccessful());
-
- assertTrue(LauncherMapperHelper.hasIdSwap(launcherJob));
+ Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
+ new XConfiguration());
+ assertTrue(LauncherMapperHelper.hasIdSwap(actionData));
MapReduceActionExecutor ae = new MapReduceActionExecutor();
ae.check(context, context.getAction());
@@ -791,8 +795,9 @@ public class TestMapReduceActionExecutor
}
});
assertTrue(launcherJob.isSuccessful());
-
- assertTrue(LauncherMapperHelper.hasIdSwap(launcherJob));
+ Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
+ new XConfiguration());
+ assertTrue(LauncherMapperHelper.hasIdSwap(actionData));
MapReduceActionExecutor ae = new MapReduceActionExecutor();
ae.check(context, context.getAction());
@@ -873,7 +878,9 @@ public class TestMapReduceActionExecutor
});
assertTrue(launcherJob.isSuccessful());
- assertTrue(LauncherMapperHelper.hasIdSwap(launcherJob));
+ Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
+ new XConfiguration());
+ assertTrue(LauncherMapperHelper.hasIdSwap(actionData));
// Assert launcher job name has been set
System.out.println("Launcher job name: " + launcherJob.getJobName());
assertTrue(launcherJob.getJobName().equals(launcherJobName));
Modified: oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java?rev=1518326&r1=1518325&r2=1518326&view=diff
==============================================================================
--- oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java (original)
+++ oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java Wed Aug 28 18:48:12 2013
@@ -22,13 +22,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.oozie.util.XConfiguration;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
-import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
-import java.util.Properties;
public class TestStreamingMain extends MainTestCase {
@@ -65,21 +62,16 @@ public class TestStreamingMain extends M
jobConf.writeXml(os);
os.close();
- File newIdProperties = new File(getTestCaseDir(), "newId.properties");
+ File newId = new File(getTestCaseDir(), "newId");
System.setProperty("oozie.action.conf.xml", actionXml.getAbsolutePath());
- System.setProperty("oozie.action.newId.properties", newIdProperties.getAbsolutePath());
+ System.setProperty("oozie.action.newId", newId.getAbsolutePath());
StreamingMain.main(null);
- assertTrue(newIdProperties.exists());
+ assertTrue(newId.exists());
+ assertNotNull(LauncherMapper.getLocalFileContentStr(newId, "", -1));
- InputStream is = new FileInputStream(newIdProperties);
- Properties props = new Properties();
- props.load(is);
- is.close();
-
- assertTrue(props.containsKey("id"));
return null;
}