You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by tu...@apache.org on 2012/06/18 22:38:14 UTC
svn commit: r1351480 - in /incubator/oozie/trunk:
core/src/test/java/org/apache/oozie/util/TestLogStreamer.java release-log.txt
Author: tucu
Date: Mon Jun 18 20:38:14 2012
New Revision: 1351480
URL: http://svn.apache.org/viewvc?rev=1351480&view=rev
Log:
OOZIE-867 Unit test to account for log retrieval from multiple gzipped oozie.log files (mona via tucu)
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java
incubator/oozie/trunk/release-log.txt
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java?rev=1351480&r1=1351479&r2=1351480&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java Mon Jun 18 20:38:14 2012
@@ -31,6 +31,10 @@ import org.apache.oozie.test.XTestCase;
import org.apache.oozie.util.XLogStreamer;
public class TestLogStreamer extends XTestCase {
+
+ static String logStatement = " - USER[oozie] GROUP[-] TOKEN[-] APP[-] "
+ + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] ";
+
public void testStreamLog() throws IOException {
long currTime = System.currentTimeMillis();
XLogStreamer.Filter.reset();
@@ -48,10 +52,8 @@ public class TestLogStreamer extends XTe
// between the start and end times of the job
FileWriter fw1 = new FileWriter(getTestCaseDir() + "/oozie.log");
StringBuilder sb1 = new StringBuilder();
- sb1.append("2009-06-24 02:43:13,958 DEBUG _L1_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
- + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change");
- sb1.append("\n2009-06-24 02:43:13,961 INFO _L2_:317 - USER[-] GROUP[-] TOKEN[-] " + "APP[example-forkjoinwf] "
- + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] "
+ sb1.append("2009-06-24 02:43:13,958 DEBUG _L1_:323" + logStatement + "End workflow state change");
+ sb1.append("\n2009-06-24 02:43:13,961 INFO _L2_:317" + logStatement
+ "[org.apache.oozie.core.command.WorkflowRunnerCallable] " + "released lock");
fw1.write(sb1.toString());
fw1.close();
@@ -62,12 +64,10 @@ public class TestLogStreamer extends XTe
// between the start and end times of the job
FileWriter fw2 = new FileWriter(getTestCaseDir() + "/oozie.log.1");
StringBuilder sb2 = new StringBuilder();
- sb2.append("\n2009-06-24 02:43:13,986 WARN _L3_:539 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
- + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] Use GenericOptionsParser for parsing " + "the "
+ sb2.append("\n2009-06-24 02:43:13,986 WARN _L3_:539" + logStatement + "Use GenericOptionsParser for parsing " + "the "
+ "arguments. " + "\n" + "_L3A_Applications "
+ "should implement Tool for the same. \n_L3B_Multi line test");
- sb2.append("\n2009-06-24 02:43:14,431 INFO _L4_:661 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
- + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] No job jar file set. User classes "
+ sb2.append("\n2009-06-24 02:43:14,431 INFO _L4_:661" + logStatement + "No job jar file set. User classes "
+ "may not be found. " + "See JobConf(Class) or JobConf#setJar(String).");
fw2.write(sb2.toString());
fw2.close();
@@ -82,8 +82,7 @@ public class TestLogStreamer extends XTe
+ "ACTION[-] Released Lock");
sb3.append("\n2009-06-24 02:43:19,344 DEBUG _L6_:323 - USER[oozie] GROUP[oozie] TOKEN[MYtoken] APP[-] JOB[-] "
+ "ACTION[-] Number of pending signals to check [0]");
- sb3.append("\n2009-06-24 02:43:29,151 DEBUG _L7_:323 - USER[-] GROUP[-] TOKEN[-] APP[-] "
- + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] Number of pending actions [0] ");
+ sb3.append("\n2009-06-24 02:43:29,151 DEBUG _L7_:323" + logStatement + "Number of pending actions [0] ");
fw3.write(sb3.toString());
fw3.close();
File f3 = new File(getTestCaseDir() + "/oozie.log.2");
@@ -94,10 +93,8 @@ public class TestLogStreamer extends XTe
// "oozie.log"
FileWriter fwerr = new FileWriter(getTestCaseDir() + "/testerr.log");
StringBuilder sberr = new StringBuilder();
- sberr.append("2009-06-24 02:43:13,958 WARN _L1_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
- + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change");
- sberr.append("\n2009-06-24 02:43:13,961 INFO _L2_:317 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
- + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] "
+ sberr.append("2009-06-24 02:43:13,958 WARN _L1_:323" + logStatement + "End workflow state change");
+ sberr.append("\n2009-06-24 02:43:13,961 INFO _L2_:317" + logStatement
+ "[org.apache.oozie.core.command.WorkflowRunnerCallable] " + "released lock");
fwerr.write(sberr.toString());
fwerr.close();
@@ -111,53 +108,32 @@ public class TestLogStreamer extends XTe
String outFilename = "oozie.log." + cal.get(Calendar.YEAR) + "-" + (cal.get(Calendar.MONTH) + 1) + "-"
+ cal.get(Calendar.DATE) + "-" + cal.get(Calendar.HOUR_OF_DAY) + ".gz";
File f = new File(getTestCaseDir() + "/" + outFilename);
- GZIPOutputStream gzout = new GZIPOutputStream(new FileOutputStream(f));
StringBuilder sb = new StringBuilder();
- sb.append("\n2009-06-24 02:43:13,958 DEBUG _L8_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-"
- + "forkjoinwf] " + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change");
- sb.append("\n2009-06-24 02:43:13,961 INFO _L9_:317 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
- + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] [org.apache.oozie.core."
+ sb.append("\n2009-06-24 02:43:13,958 DEBUG _L8_:323" + logStatement + "End workflow state change");
+ sb.append("\n2009-06-24 02:43:13,961 INFO _L9_:317" + logStatement + "[org.apache.oozie.core."
+ "command.WorkflowRunnerCallable] " + "released lock");
- String strg = sb.toString();
- byte[] buf = strg.getBytes();
- gzout.write(buf, 0, buf.length);
- gzout.finish();
- gzout.close();
+ writeToGZFile(f,sb);
// oozie.log.gz GZip file would always be included in list of files for log retrieval
outFilename = "oozie.log.gz";
f = new File(getTestCaseDir() + "/" + outFilename);
- gzout = new GZIPOutputStream(new FileOutputStream(f));
// Generate and write log content to the GZip file
sb = new StringBuilder();
- sb.append("\n2009-06-24 02:43:13,958 DEBUG _L10_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-"
- + "forkjoinwf] " + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change");
- sb.append("\n2009-06-24 02:43:13,961 INFO _L11_:317 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
- + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] [org.apache.oozie.core."
+ sb.append("\n2009-06-24 02:43:13,958 DEBUG _L10_:323" + logStatement + "End workflow state change");
+ sb.append("\n2009-06-24 02:43:13,961 INFO _L11_:317" + logStatement + "[org.apache.oozie.core."
+ "command.WorkflowRunnerCallable] " + "released lock");
- strg = sb.toString();
- buf = strg.getBytes();
- gzout.write(buf, 0, buf.length);
- gzout.finish();
- gzout.close();
+ writeToGZFile(f,sb);
// Test to check if an invalid GZip file(file name not in the expected format oozie.log-YYYY-MM-DD-HH.gz) is
// excluded from log retrieval
outFilename = "oozie.log-2011-12-03-15.bz2.gz";
f = new File(getTestCaseDir() + "/" + outFilename);
- gzout = new GZIPOutputStream(new FileOutputStream(f));
// Generate and write log content to the GZip file
sb = new StringBuilder();
- sb.append("\n2009-06-24 02:43:13,958 DEBUG _L12_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-"
- + "forkjoinwf] " + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change");
- sb.append("\n2009-06-24 02:43:13,961 INFO _L13_:317 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
- + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] [org.apache.oozie.core."
+ sb.append("\n2009-06-24 02:43:13,958 DEBUG _L12_:323" + logStatement + "End workflow state change");
+ sb.append("\n2009-06-24 02:43:13,961 INFO _L13_:317" + logStatement + "[org.apache.oozie.core."
+ "command.WorkflowRunnerCallable] " + "released lock");
- strg = sb.toString();
- buf = strg.getBytes();
- gzout.write(buf, 0, buf.length);
- gzout.finish();
- gzout.close();
+ writeToGZFile(f,sb);
// Test for the log retrieval of the job that began 10 seconds before and ended 5 seconds before current time
// respectively
@@ -196,4 +172,79 @@ public class TestLogStreamer extends XTe
assertEquals(true, out[6].contains("_L4_"));
assertEquals(true, out[7].contains("_L7_"));
}
+
+ public void testStreamLogMultipleHours() throws IOException {
+
+ long currTime = System.currentTimeMillis();
+ XLogStreamer.Filter.reset();
+ XLogStreamer.Filter.defineParameter("USER");
+ XLogStreamer.Filter.defineParameter("GROUP");
+ XLogStreamer.Filter.defineParameter("TOKEN");
+ XLogStreamer.Filter.defineParameter("APP");
+ XLogStreamer.Filter.defineParameter("JOB");
+ XLogStreamer.Filter.defineParameter("ACTION");
+ XLogStreamer.Filter xf = new XLogStreamer.Filter();
+ xf.setParameter("JOB", "14-200904160239--example-forkjoinwf");
+ xf.setLogLevel("DEBUG|INFO");
+
+ // Test to check if all gz log files in the range jobStartTime-currentTime are retrieved
+ String outFilename = "oozie.log.2012-04-24-19.gz";
+ File f = new File(getTestCaseDir() + "/" + outFilename);
+ StringBuilder logLines = new StringBuilder();
+ logLines.append("\n2012-04-24 19:43:13,958 DEBUG _L19_:323" + logStatement);
+ writeToGZFile(f,logLines);
+
+ outFilename = "oozie.log.2012-04-24-20.gz";
+ f = new File(getTestCaseDir() + "/" + outFilename);
+ logLines = new StringBuilder();
+ logLines.append("\n2012-04-24 20:43:13,958 DEBUG _L20_:323" + logStatement);
+ writeToGZFile(f,logLines);
+
+ outFilename = "oozie.log.2012-04-24-21.gz";
+ f = new File(getTestCaseDir() + "/" + outFilename);
+ logLines = new StringBuilder();
+ logLines.append("\n2012-04-24 21:43:13,958 DEBUG _L21_:323" + logStatement);
+ writeToGZFile(f,logLines);
+
+ // This file will be always included in the list of files for log retrieval, provided the modification time lies
+ // between the start and end times of the job
+ FileWriter fw1 = new FileWriter(getTestCaseDir() + "/oozie.log");
+ StringBuilder sb1 = new StringBuilder();
+ sb1.append("\n2012-04-24 22:43:13,958 DEBUG _L22_:323" + logStatement);
+ sb1.append("\n2012-04-24 22:43:13,961 INFO _L23_:317" + logStatement
+ + "[org.apache.oozie.core.command.WorkflowRunnerCallable] " + "released lock");
+ fw1.write(sb1.toString());
+ fw1.close();
+ File f1 = new File(getTestCaseDir() + "/oozie.log");
+ f1.setLastModified(currTime - 5000);
+
+ // Test for the log retrieval of the job spanning multiple hours
+ StringWriter sw2 = new StringWriter();
+ XLogStreamer str2 = new XLogStreamer(xf, sw2, getTestCaseDir(), "oozie.log", 1);
+ Calendar calendarEntry = Calendar.getInstance();
+ // Setting start-time to 2012-04-24-19 for log stream (month-1 passed as parameter since 0=January), and end time is current time
+ calendarEntry.set(2012, 3, 24, 19, 0);
+ str2.streamLog(calendarEntry.getTime(), new Date(System.currentTimeMillis()));
+ String[] out = sw2.toString().split("\n");
+
+ // Check if the retrieved log content is of length five lines after filtering based on time window, file name
+ // pattern and parameters like JobId, Username etc. and/or based on log level like INFO, DEBUG, etc.
+ assertEquals(5, out.length);
+ // Check if the lines of the log contain the expected strings
+ assertEquals(true, out[0].contains("_L19_"));
+ assertEquals(true, out[1].contains("_L20_"));
+ assertEquals(true, out[2].contains("_L21_"));
+ // from oozie.log
+ assertEquals(true, out[3].contains("_L22_"));
+ assertEquals(true, out[4].contains("_L23_"));
+ }
+
+ private void writeToGZFile(File f, StringBuilder sbr) throws IOException {
+ GZIPOutputStream gzout = new GZIPOutputStream(new FileOutputStream(f));
+ String strg = sbr.toString();
+ // Write log content to the GZip file
+ byte[] buf = strg.getBytes();
+ gzout.write(buf, 0, buf.length);
+ gzout.close();
+ }
}
Modified: incubator/oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1351480&r1=1351479&r2=1351480&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Mon Jun 18 20:38:14 2012
@@ -1,5 +1,6 @@
-- Oozie 3.3.0 release (trunk - unreleased)
+OOZIE-867 Unit test to account for log retrieval from multiple gzipped oozie.log files (mona via tucu)
OOZIE-649 Fail fast when a date doesn't parse correctly (rkanter via tucu)
OOZIE-871 actions from subworkflows that use oozie.libpath have duplicate classpath (tucu)
OOZIE-764 remove log warnings when credentials are null (rkanter via tucu)