You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by an...@apache.org on 2012/01/06 22:58:08 UTC
svn commit: r1228445 - in /incubator/oozie/branches/3.1:
core/src/main/java/org/apache/oozie/util/XLogReader.java
core/src/main/java/org/apache/oozie/util/XLogStreamer.java
core/src/test/java/org/apache/oozie/util/TestLogStreamer.java release-log.txt
Author: angeloh
Date: Fri Jan 6 21:58:07 2012
New Revision: 1228445
URL: http://svn.apache.org/viewvc?rev=1228445&view=rev
Log:
OOZIE-26 Ability to get the log content from Archived file(.gz format).(Kiran N via Mona via Mohammad)
Modified:
incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/util/XLogReader.java
incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/util/XLogStreamer.java
incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java
incubator/oozie/branches/3.1/release-log.txt
Modified: incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/util/XLogReader.java
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/util/XLogReader.java?rev=1228445&r1=1228444&r2=1228445&view=diff
==============================================================================
--- incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/util/XLogReader.java (original)
+++ incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/util/XLogReader.java Fri Jan 6 21:58:07 2012
@@ -78,4 +78,8 @@ public class XLogReader {
}
logWriter.flush();
}
+
+ public void close() throws IOException {
+ logReader.close();
+ }
}
Modified: incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/util/XLogStreamer.java
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/util/XLogStreamer.java?rev=1228445&r1=1228444&r2=1228445&view=diff
==============================================================================
--- incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/util/XLogStreamer.java (original)
+++ incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/util/XLogStreamer.java Fri Jan 6 21:58:07 2012
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.Writer;
import java.util.ArrayList;
+import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
@@ -30,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
/**
* XLogStreamer streams the given log file to logWriter after applying the given filter.
@@ -158,21 +160,6 @@ public class XLogStreamer {
else {
sb.append("(.* - ");
for (int i = 0; i < parameters.size(); i++) {
- if (parameters.get(i).equals("ACTION")) {
- String[] actionsList = filterParams.get(parameters.get(i)).split(",");
- sb.append("(");
-
- for (int counter = 0; counter < actionsList.length; counter++) {
- if (counter != 0) {
- sb.append("|");
- }
- sb.append("ACTION" + "\\[");
- sb.append(actionsList[counter] + "\\]");
- }
-
- sb.append(") ");
- continue;
- }
sb.append(parameters.get(i) + "\\[");
sb.append(filterParams.get(parameters.get(i)) + "\\] ");
}
@@ -214,10 +201,6 @@ public class XLogStreamer {
public void streamLog(Date startTime, Date endTime) throws IOException {
long startTimeMillis = 0;
long endTimeMillis;
- String fileName;
- InputStream ifs;
- XLogReader logReader;
-
if (startTime != null) {
startTimeMillis = startTime.getTime();
}
@@ -229,11 +212,25 @@ public class XLogStreamer {
}
File dir = new File(logPath);
ArrayList<FileInfo> fileList = getFileList(dir, startTimeMillis, endTimeMillis, logRotation, logFile);
+ File file;
+ String fileName;
+ XLogReader logReader;
for (int i = 0; i < fileList.size(); i++) {
fileName = fileList.get(i).getFileName();
- ifs = new FileInputStream(fileList.get(i).getFileName());
+ if (fileName.endsWith(".gz")) {
+ file = new File(fileName);
+ GZIPInputStream gzipInputStream = null;
+ gzipInputStream = new GZIPInputStream(new FileInputStream(file));
+ logReader = new XLogReader(gzipInputStream, logFilter, logWriter);
+ logReader.processLog();
+ logReader.close();
+ continue;
+ }
+ InputStream ifs;
+ ifs = new FileInputStream(fileName);
logReader = new XLogReader(ifs, logFilter, logWriter);
logReader.processLog();
+ ifs.close();
}
}
@@ -283,21 +280,24 @@ public class XLogStreamer {
*/
private ArrayList<FileInfo> getFileList(File dir, long startTime, long endTime, long logRotationTime, String logFile) {
String[] children = dir.list();
- String fileName;
- File file;
ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
-
if (children == null) {
return fileList;
}
else {
for (int i = 0; i < children.length; i++) {
- fileName = children[i];
- file = new File(dir.getAbsolutePath(), fileName);
+ String fileName = children[i];
if (!fileName.startsWith(logFile) && !fileName.equals(logFile)) {
continue;
}
- file = new File(dir.getAbsolutePath(), fileName);
+ File file = new File(dir.getAbsolutePath(), fileName);
+ if (fileName.endsWith(".gz")) {
+ long gzFileCreationTime = getGZFileCreationTime(fileName, startTime, endTime);
+ if (gzFileCreationTime != -1) {
+ fileList.add(new FileInfo(file.getAbsolutePath(), gzFileCreationTime));
+ }
+ continue;
+ }
long modTime = file.lastModified();
if (modTime < startTime) {
continue;
@@ -311,4 +311,34 @@ public class XLogStreamer {
Collections.sort(fileList);
return fileList;
}
+
+ /**
+ * Returns the creation time of the .gz archive if it is relevant to the job
+ *
+ * @param fileName
+ * @param startTime
+ * @param endTime
+ * @return Modification time of .gz file after checking if it is relevant to the job
+ */
+ private long getGZFileCreationTime(String fileName, long startTime, long endTime) {
+ long returnVal = -1;
+ int dateStartIndex = 10;
+ String[] dateDetails;
+ dateDetails = fileName.substring(dateStartIndex, fileName.length() - 3).split("-");
+ int year = Integer.parseInt(dateDetails[0]);
+ int month = Integer.parseInt(dateDetails[1]);
+ int day = Integer.parseInt(dateDetails[2]);
+ int hour = Integer.parseInt(dateDetails[3]);
+ int minute = 0;
+ Calendar calendarEntry = Calendar.getInstance();
+ calendarEntry.set(year, month - 1, day, hour, minute); // give month-1(Say, 7 for August)
+ long logFileStartTime = calendarEntry.getTimeInMillis();
+ long milliSecondsPerHour = 3600000;
+ long logFileEndTime = logFileStartTime + milliSecondsPerHour;
+ if ((startTime >= logFileStartTime && startTime <= logFileEndTime)
+ || (endTime >= logFileStartTime && endTime <= logFileEndTime)) {
+ returnVal = logFileStartTime;
+ }
+ return returnVal;
+ }
}
Modified: incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java?rev=1228445&r1=1228444&r2=1228445&view=diff
==============================================================================
--- incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java (original)
+++ incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java Fri Jan 6 21:58:07 2012
@@ -18,15 +18,18 @@
package org.apache.oozie.util;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringWriter;
+import java.util.Calendar;
import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.zip.GZIPOutputStream;
import org.apache.oozie.test.XTestCase;
import org.apache.oozie.util.XLogStreamer;
-
public class TestLogStreamer extends XTestCase {
public void testStreamLog() throws IOException {
long currTime = System.currentTimeMillis();
@@ -41,60 +44,95 @@ public class TestLogStreamer extends XTe
xf.setParameter("JOB", "14-200904160239--example-forkjoinwf");
xf.setLogLevel("DEBUG|INFO");
- FileWriter fw1 = new FileWriter(getTestCaseDir() + "/test.log");
+ FileWriter fw1 = new FileWriter(getTestCaseDir() + "/oozie.log");
StringBuilder sb1 = new StringBuilder();
- sb1.append("2009-06-24 02:43:13,958 DEBUG _L1_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change");
- sb1.append("\n2009-06-24 02:43:13,961 INFO _L2_:317 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] [org.apache.oozie.core.command.WorkflowRunnerCallable] released lock");
+ sb1.append("2009-06-24 02:43:13,958 DEBUG _L1_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
+ + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change");
+ sb1.append("\n2009-06-24 02:43:13,961 INFO _L2_:317 - USER[-] GROUP[-] TOKEN[-] " + "APP[example-forkjoinwf] "
+ + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] "
+ + "[org.apache.oozie.core.command.WorkflowRunnerCallable] " + "released lock");
fw1.write(sb1.toString());
fw1.close();
- File f1 = new File(getTestCaseDir() + "/test.log");
+ File f1 = new File(getTestCaseDir() + "/oozie.log");
f1.setLastModified(currTime - 9000);
- FileWriter fw2 = new FileWriter(getTestCaseDir() + "/test.log.1");
+ FileWriter fw2 = new FileWriter(getTestCaseDir() + "/oozie.log.1");
StringBuilder sb2 = new StringBuilder();
- sb2.append("\n2009-06-24 02:43:13,986 WARN _L3_:539 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] Use GenericOptionsParser for parsing the arguments. \n_L3A_Applications should implement Tool for the same. \n_L3B_Multi line test");
- sb2.append("\n2009-06-24 02:43:14,431 INFO _L4_:661 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] No job jar file set. User classes may not be found. See JobConf(Class) or JobConf#setJar(String).");
+ sb2.append("\n2009-06-24 02:43:13,986 WARN _L3_:539 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
+ + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] Use GenericOptionsParser for parsing " + "the "
+ + "arguments. " + "\n" + "_L3A_Applications "
+ + "should implement Tool for the same. \n_L3B_Multi line test");
+ sb2.append("\n2009-06-24 02:43:14,431 INFO _L4_:661 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
+ + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] No job jar file set. User classes "
+ + "may not be found. " + "See JobConf(Class) or JobConf#setJar(String).");
fw2.write(sb2.toString());
fw2.close();
- File f2 = new File(getTestCaseDir() + "/test.log.1");
+ File f2 = new File(getTestCaseDir() + "/oozie.log.1");
f2.setLastModified(currTime - 8000);
- FileWriter fw3 = new FileWriter(getTestCaseDir() + "/test.log.2");
+ FileWriter fw3 = new FileWriter(getTestCaseDir() + "/oozie.log.2");
StringBuilder sb3 = new StringBuilder();
- sb3.append("\n2009-06-24 02:43:14,505 INFO _L5_:317 - USER[oozie] GROUP[oozie] TOKEN[-] APP[-] JOB[-] ACTION[-] Released Lock");
- sb3.append("\n2009-06-24 02:43:19,344 DEBUG _L6_:323 - USER[oozie] GROUP[oozie] TOKEN[MYtoken] APP[-] JOB[-] ACTION[-] Number of pending signals to check [0]");
- sb3.append("\n2009-06-24 02:43:29,151 DEBUG _L7_:323 - USER[-] GROUP[-] TOKEN[-] APP[-] JOB[14-200904160239--example-forkjoinwf] ACTION[-] Number of pending actions [0] ");
+ sb3.append("\n2009-06-24 02:43:14,505 INFO _L5_:317 - USER[oozie] GROUP[oozie] TOKEN[-] APP[-] JOB[-] "
+ + "ACTION[-] Released Lock");
+ sb3.append("\n2009-06-24 02:43:19,344 DEBUG _L6_:323 - USER[oozie] GROUP[oozie] TOKEN[MYtoken] APP[-] JOB[-] "
+ + "ACTION[-] Number of pending signals to check [0]");
+ sb3.append("\n2009-06-24 02:43:29,151 DEBUG _L7_:323 - USER[-] GROUP[-] TOKEN[-] APP[-] "
+ + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] Number of pending actions [0] ");
fw3.write(sb3.toString());
fw3.close();
- File f3 = new File(getTestCaseDir() + "/test.log.2");
+ File f3 = new File(getTestCaseDir() + "/oozie.log.2");
f3.setLastModified(currTime);
FileWriter fwerr = new FileWriter(getTestCaseDir() + "/testerr.log");
StringBuilder sberr = new StringBuilder();
- sberr.append("2009-06-24 02:43:13,958 WARN _L1_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change");
- sberr.append("\n2009-06-24 02:43:13,961 INFO _L2_:317 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] [org.apache.oozie.core.command.WorkflowRunnerCallable] released lock");
+ sberr.append("2009-06-24 02:43:13,958 WARN _L1_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
+ + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change");
+ sberr.append("\n2009-06-24 02:43:13,961 INFO _L2_:317 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
+ + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] "
+ + "[org.apache.oozie.core.command.WorkflowRunnerCallable] " + "released lock");
fwerr.write(sberr.toString());
fwerr.close();
File ferr = new File(getTestCaseDir() + "/testerr.log");
ferr.setLastModified(currTime - 8000);
+ Calendar cal = new GregorianCalendar();
+ String outFilename = "oozie.log." + cal.get(Calendar.YEAR) + "-" + (cal.get(Calendar.MONTH) + 1) + "-"
+ + cal.get(Calendar.DATE) + "-" + cal.get(Calendar.HOUR_OF_DAY) + ".gz";
+ File f = new File(getTestCaseDir() + "/" + outFilename);
+ GZIPOutputStream gzout = new GZIPOutputStream(new FileOutputStream(f));
+ StringBuilder sb = new StringBuilder();
+ sb.append("\n2009-06-24 02:43:13,958 DEBUG _L8_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-"
+ + "forkjoinwf] " + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change");
+ sb.append("\n2009-06-24 02:43:13,961 INFO _L9_:317 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
+ + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] [org.apache.oozie.core."
+ + "command.WorkflowRunnerCallable] " + "released lock");
+ String strg = sb.toString();
+ byte[] buf = strg.getBytes();
+ gzout.write(buf, 0, buf.length);
+ gzout.finish();
+ gzout.close();
+
StringWriter sw = new StringWriter();
- XLogStreamer str = new XLogStreamer(xf, sw, getTestCaseDir(), "test.log", 1);
+ XLogStreamer str = new XLogStreamer(xf, sw, getTestCaseDir(), "oozie.log", 1);
str.streamLog(new Date(currTime - 10000), new Date(currTime - 5000));
String[] out = sw.toString().split("\n");
- assertEquals(3, out.length);
- assertEquals(true, out[0].contains("_L1_"));
- assertEquals(true, out[1].contains("_L2_"));
- assertEquals(true, out[2].contains("_L4_"));
+ assertEquals(5, out.length);
+ assertEquals(true, out[0].contains("_L8_"));
+ assertEquals(true, out[1].contains("_L9_"));
+ assertEquals(true, out[2].contains("_L1_"));
+ assertEquals(true, out[3].contains("_L2_"));
+ assertEquals(true, out[4].contains("_L4_"));
StringWriter sw1 = new StringWriter();
- XLogStreamer str1 = new XLogStreamer(xf, sw1, getTestCaseDir(), "test.log", 1);
+ XLogStreamer str1 = new XLogStreamer(xf, sw1, getTestCaseDir(), "oozie.log", 1);
str1.streamLog(null, null);
out = sw1.toString().split("\n");
- assertEquals(4, out.length);
- assertEquals(true, out[0].contains("_L1_"));
- assertEquals(true, out[1].contains("_L2_"));
- assertEquals(true, out[2].contains("_L4_"));
- assertEquals(true, out[3].contains("_L7_"));
+ assertEquals(6, out.length);
+ assertEquals(true, out[0].contains("_L8_"));
+ assertEquals(true, out[1].contains("_L9_"));
+ assertEquals(true, out[2].contains("_L1_"));
+ assertEquals(true, out[3].contains("_L2_"));
+ assertEquals(true, out[4].contains("_L4_"));
+ assertEquals(true, out[5].contains("_L7_"));
}
}
Modified: incubator/oozie/branches/3.1/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/release-log.txt?rev=1228445&r1=1228444&r2=1228445&view=diff
==============================================================================
--- incubator/oozie/branches/3.1/release-log.txt (original)
+++ incubator/oozie/branches/3.1/release-log.txt Fri Jan 6 21:58:07 2012
@@ -1,4 +1,5 @@
-- Oozie 3.1.1 release
+OOZIE-26 Ability to get the log content from Archived file(.gz format)
OOZIE-554 New filters for all kinds of jobs
OOZIE-556 Sort on Web Console for non-string data type
OOZIE-41 Usability improvement (Additional options) for coordinator job analysis on Web Console