You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2013/02/04 18:57:14 UTC
svn commit: r1442212 - in /oozie/branches/branch-3.3: ./
core/src/main/java/org/apache/oozie/util/
core/src/test/java/org/apache/oozie/service/
core/src/test/java/org/apache/oozie/util/ core/src/test/resources/
Author: rkanter
Date: Mon Feb 4 17:57:13 2013
New Revision: 1442212
URL: http://svn.apache.org/viewvc?rev=1442212&view=rev
Log:
OOZIE-999 XLogStreamer requires log4j.appender.oozie.layout.ConversionPattern to contain a - (dash) or streaming logs to web UI and CLI don't work (rkanter via virag)
Added:
oozie/branches/branch-3.3/core/src/test/resources/test-no-dash-log4j.properties
Modified:
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/util/XLogStreamer.java
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/service/TestXLogService.java
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java
oozie/branches/branch-3.3/release-log.txt
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/util/XLogStreamer.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/util/XLogStreamer.java?rev=1442212&r1=1442211&r2=1442212&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/util/XLogStreamer.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/util/XLogStreamer.java Mon Feb 4 17:57:13 2013
@@ -159,7 +159,7 @@ public class XLogStreamer {
sb.append("(.*)");
}
else {
- sb.append("(.* - ");
+ sb.append("(.* ");
for (int i = 0; i < parameters.size(); i++) {
sb.append(parameters.get(i) + "\\[");
sb.append(filterParams.get(parameters.get(i)) + "\\] ");
Modified: oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/service/TestXLogService.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/service/TestXLogService.java?rev=1442212&r1=1442211&r2=1442212&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/service/TestXLogService.java (original)
+++ oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/service/TestXLogService.java Mon Feb 4 17:57:13 2013
@@ -29,6 +29,7 @@ import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.StringWriter;
import java.util.Properties;
+import org.apache.oozie.util.XLogStreamer;
public class TestXLogService extends XTestCase {
@@ -267,14 +268,61 @@ public class TestXLogService extends XTe
assertFalse(doStreamDisabledCheck(ls));
ls.destroy();
}
-
+
+ public void testNoDashInConversionPattern() throws Exception{
+ XLogStreamer.Filter.reset();
+ XLogStreamer.Filter.defineParameter("USER");
+ XLogStreamer.Filter.defineParameter("GROUP");
+ XLogStreamer.Filter.defineParameter("TOKEN");
+ XLogStreamer.Filter.defineParameter("APP");
+ XLogStreamer.Filter.defineParameter("JOB");
+ XLogStreamer.Filter.defineParameter("ACTION");
+ XLogStreamer.Filter xf = new XLogStreamer.Filter();
+ xf.setParameter("USER", "oozie");
+ xf.setLogLevel("DEBUG|INFO");
+ // Previously, a dash ("-") was always required somewhere in a line in order for that line to pass the filter; this test
+ // checks that this condition is no longer required for log streaming to work
+ File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties");
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties");
+ IOUtils.copyStream(is, new FileOutputStream(log4jFile));
+ setSystemProperty(XLogService.LOG4J_FILE, "test-log4j.properties");
+ final XLogService ls = new XLogService();
+ ls.init(null);
+ assertFalse((doStreamDisabledCheck(ls)));
+ LogFactory.getLog("a").info("2009-06-24 02:43:14,505 INFO _L1_:317 - USER[oozie] GROUP[oozie] TOKEN[-] APP[-] JOB[-] "
+ + "ACTION[-] Released Lock");
+ LogFactory.getLog("a").info("2009-06-24 02:43:14,505 INFO _L2_:317 - USER[blah] GROUP[oozie] TOKEN[-] APP[-] JOB[-] "
+ + "ACTION[-] Released Lock");
+ LogFactory.getLog("a").info("2009-06-24 02:43:14,505 INFO _L3_:317 USER[oozie] GROUP[oozie] TOKEN[-] APP[-] JOB[-] "
+ + "ACTION[-] Released Lock");
+ LogFactory.getLog("a").info("2009-06-24 02:43:14,505 INFO _L4_:317 USER[blah] GROUP[oozie] TOKEN[-] APP[-] JOB[-] "
+ + "ACTION[-] Released Lock");
+ String out = doStreamLog(ls, xf);
+ String outArr[] = out.split("\n");
+ // Lines 2 and 4 are filtered out because they have the wrong user
+ assertEquals(2, outArr.length);
+ assertTrue(outArr[0].contains("_L1_"));
+ assertFalse(out.contains("_L2_"));
+ assertTrue(outArr[1].contains("_L3_"));
+ assertFalse(out.contains("_L4_"));
+ ls.destroy();
+ }
+
private boolean doStreamDisabledCheck(XLogService ls) throws Exception {
- StringWriter w = new StringWriter();
- ls.streamLog(null, null, null, w);
- if (w.toString().equals("Log streaming disabled!!")) {
+ if (doStreamLog(ls).equals("Log streaming disabled!!")) {
return true;
}
return false;
}
+ private String doStreamLog(XLogService ls) throws Exception {
+ return doStreamLog(ls, null);
+ }
+
+ private String doStreamLog(XLogService ls, XLogStreamer.Filter xf) throws Exception {
+ StringWriter w = new StringWriter();
+ ls.streamLog(xf, null, null, w);
+ return w.toString();
+ }
}
Modified: oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java?rev=1442212&r1=1442211&r2=1442212&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java (original)
+++ oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java Mon Feb 4 17:57:13 2013
@@ -33,7 +33,7 @@ import org.apache.oozie.util.XLogStreame
public class TestLogStreamer extends XTestCase {
- static String logStatement = " - USER[oozie] GROUP[-] TOKEN[-] APP[-] "
+ static String logStatement = " USER[oozie] GROUP[-] TOKEN[-] APP[-] "
+ "JOB[14-200904160239--example-forkjoinwf] ACTION[-] ";
private final static SimpleDateFormat filenameDateFormatter = new SimpleDateFormat("yyyy-MM-dd-HH");
@@ -55,8 +55,8 @@ public class TestLogStreamer extends XTe
// between the start and end times of the job
FileWriter fw1 = new FileWriter(getTestCaseDir() + "/oozie.log");
StringBuilder sb1 = new StringBuilder();
- sb1.append("2009-06-24 02:43:13,958 DEBUG _L1_:323" + logStatement + "End workflow state change");
- sb1.append("\n2009-06-24 02:43:13,961 INFO _L2_:317" + logStatement
+ sb1.append("2009-06-24 02:43:13,958 DEBUG _L1_:323 -" + logStatement + "End workflow state change");
+ sb1.append("\n2009-06-24 02:43:13,961 INFO _L2_:317 -" + logStatement
+ "[org.apache.oozie.core.command.WorkflowRunnerCallable] " + "released lock");
fw1.write(sb1.toString());
fw1.close();
@@ -67,10 +67,10 @@ public class TestLogStreamer extends XTe
// between the start and end times of the job
FileWriter fw2 = new FileWriter(getTestCaseDir() + "/oozie.log.1");
StringBuilder sb2 = new StringBuilder();
- sb2.append("\n2009-06-24 02:43:13,986 WARN _L3_:539" + logStatement + "Use GenericOptionsParser for parsing " + "the "
+ sb2.append("\n2009-06-24 02:43:13,986 WARN _L3_:539 -" + logStatement + "Use GenericOptionsParser for parsing " + "the "
+ "arguments. " + "\n" + "_L3A_Applications "
+ "should implement Tool for the same. \n_L3B_Multi line test");
- sb2.append("\n2009-06-24 02:43:14,431 INFO _L4_:661" + logStatement + "No job jar file set. User classes "
+ sb2.append("\n2009-06-24 02:43:14,431 INFO _L4_:661 -" + logStatement + "No job jar file set. User classes "
+ "may not be found. " + "See JobConf(Class) or JobConf#setJar(String).");
fw2.write(sb2.toString());
fw2.close();
@@ -85,7 +85,7 @@ public class TestLogStreamer extends XTe
+ "ACTION[-] Released Lock");
sb3.append("\n2009-06-24 02:43:19,344 DEBUG _L6_:323 - USER[oozie] GROUP[oozie] TOKEN[MYtoken] APP[-] JOB[-] "
+ "ACTION[-] Number of pending signals to check [0]");
- sb3.append("\n2009-06-24 02:43:29,151 DEBUG _L7_:323" + logStatement + "Number of pending actions [0] ");
+ sb3.append("\n2009-06-24 02:43:29,151 DEBUG _L7_:323 -" + logStatement + "Number of pending actions [0] ");
fw3.write(sb3.toString());
fw3.close();
File f3 = new File(getTestCaseDir() + "/oozie.log.2");
@@ -96,8 +96,8 @@ public class TestLogStreamer extends XTe
// "oozie.log"
FileWriter fwerr = new FileWriter(getTestCaseDir() + "/testerr.log");
StringBuilder sberr = new StringBuilder();
- sberr.append("2009-06-24 02:43:13,958 WARN _L1_:323" + logStatement + "End workflow state change");
- sberr.append("\n2009-06-24 02:43:13,961 INFO _L2_:317" + logStatement
+ sberr.append("2009-06-24 02:43:13,958 WARN _L1_:323 -" + logStatement + "End workflow state change");
+ sberr.append("\n2009-06-24 02:43:13,961 INFO _L2_:317 -" + logStatement
+ "[org.apache.oozie.core.command.WorkflowRunnerCallable] " + "released lock");
fwerr.write(sberr.toString());
fwerr.close();
@@ -110,8 +110,8 @@ public class TestLogStreamer extends XTe
String outFilename = "oozie.log-" + filenameDateFormatter.format(new Date(currTime)) + ".gz";
File f = new File(getTestCaseDir() + "/" + outFilename);
StringBuilder sb = new StringBuilder();
- sb.append("\n2009-06-24 02:43:13,958 DEBUG _L8_:323" + logStatement + "End workflow state change");
- sb.append("\n2009-06-24 02:43:13,961 INFO _L9_:317" + logStatement + "[org.apache.oozie.core."
+ sb.append("\n2009-06-24 02:43:13,958 DEBUG _L8_:323 -" + logStatement + "End workflow state change");
+ sb.append("\n2009-06-24 02:43:13,961 INFO _L9_:317 -" + logStatement + "[org.apache.oozie.core."
+ "command.WorkflowRunnerCallable] " + "released lock");
writeToGZFile(f,sb);
@@ -120,8 +120,8 @@ public class TestLogStreamer extends XTe
f = new File(getTestCaseDir() + "/" + outFilename);
// Generate and write log content to the GZip file
sb = new StringBuilder();
- sb.append("\n2009-06-24 02:43:13,958 DEBUG _L10_:323" + logStatement + "End workflow state change");
- sb.append("\n2009-06-24 02:43:13,961 INFO _L11_:317" + logStatement + "[org.apache.oozie.core."
+ sb.append("\n2009-06-24 02:43:13,958 DEBUG _L10_:323 -" + logStatement + "End workflow state change");
+ sb.append("\n2009-06-24 02:43:13,961 INFO _L11_:317 -" + logStatement + "[org.apache.oozie.core."
+ "command.WorkflowRunnerCallable] " + "released lock");
writeToGZFile(f,sb);
@@ -131,8 +131,8 @@ public class TestLogStreamer extends XTe
f = new File(getTestCaseDir() + "/" + outFilename);
// Generate and write log content to the GZip file
sb = new StringBuilder();
- sb.append("\n2009-06-24 02:43:13,958 DEBUG _L12_:323" + logStatement + "End workflow state change");
- sb.append("\n2009-06-24 02:43:13,961 INFO _L13_:317" + logStatement + "[org.apache.oozie.core."
+ sb.append("\n2009-06-24 02:43:13,958 DEBUG _L12_:323 -" + logStatement + "End workflow state change");
+ sb.append("\n2009-06-24 02:43:13,961 INFO _L13_:317 -" + logStatement + "[org.apache.oozie.core."
+ "command.WorkflowRunnerCallable] " + "released lock");
writeToGZFile(f,sb);
@@ -192,27 +192,27 @@ public class TestLogStreamer extends XTe
String outFilename = "oozie.log-2012-04-24-19.gz";
File f = new File(getTestCaseDir() + "/" + outFilename);
StringBuilder logLines = new StringBuilder();
- logLines.append("\n2012-04-24 19:43:13,958 DEBUG _L19_:323" + logStatement);
+ logLines.append("\n2012-04-24 19:43:13,958 DEBUG _L19_:323 -" + logStatement);
writeToGZFile(f,logLines);
outFilename = "oozie.log-2012-04-24-20.gz";
f = new File(getTestCaseDir() + "/" + outFilename);
logLines = new StringBuilder();
- logLines.append("\n2012-04-24 20:43:13,958 DEBUG _L20_:323" + logStatement);
+ logLines.append("\n2012-04-24 20:43:13,958 DEBUG _L20_:323 -" + logStatement);
writeToGZFile(f,logLines);
outFilename = "oozie.log-2012-04-24-21.gz";
f = new File(getTestCaseDir() + "/" + outFilename);
logLines = new StringBuilder();
- logLines.append("\n2012-04-24 21:43:13,958 DEBUG _L21_:323" + logStatement);
+ logLines.append("\n2012-04-24 21:43:13,958 DEBUG _L21_:323 -" + logStatement);
writeToGZFile(f,logLines);
// This file will be always included in the list of files for log retrieval, provided the modification time lies
// between the start and end times of the job
FileWriter fw1 = new FileWriter(getTestCaseDir() + "/oozie.log");
StringBuilder sb1 = new StringBuilder();
- sb1.append("\n2012-04-24 22:43:13,958 DEBUG _L22_:323" + logStatement);
- sb1.append("\n2012-04-24 22:43:13,961 INFO _L23_:317" + logStatement
+ sb1.append("\n2012-04-24 22:43:13,958 DEBUG _L22_:323 -" + logStatement);
+ sb1.append("\n2012-04-24 22:43:13,961 INFO _L23_:317 -" + logStatement
+ "[org.apache.oozie.core.command.WorkflowRunnerCallable] " + "released lock");
fw1.write(sb1.toString());
fw1.close();
@@ -240,6 +240,53 @@ public class TestLogStreamer extends XTe
assertEquals(true, out[4].contains("_L23_"));
}
+ public void testStreamLogNoDash() throws IOException {
+ long currTime = System.currentTimeMillis();
+ XLogStreamer.Filter.reset();
+ XLogStreamer.Filter.defineParameter("USER");
+ XLogStreamer.Filter.defineParameter("GROUP");
+ XLogStreamer.Filter.defineParameter("TOKEN");
+ XLogStreamer.Filter.defineParameter("APP");
+ XLogStreamer.Filter.defineParameter("JOB");
+ XLogStreamer.Filter.defineParameter("ACTION");
+ XLogStreamer.Filter xf = new XLogStreamer.Filter();
+ xf.setParameter("JOB", "14-200904160239--example-forkjoinwf");
+ xf.setLogLevel("DEBUG|INFO");
+
+ // Previously, a dash ("-") was always required somewhere in a line in order for that line to pass the filter; this test
+ // checks that this condition is no longer required
+ FileWriter fw1 = new FileWriter(getTestCaseDir() + "/oozie.log");
+ StringBuilder sb1 = new StringBuilder();
+ sb1.append("2009-06-24 02:43:13,958 DEBUG _L1_:323 -" + logStatement + "End workflow state change\n");
+ sb1.append("2009-06-24 02:43:13,958 DEBUG _L2_:323 +" + logStatement + "End workflow state change\n");
+ sb1.append("2009-06-24 02:43:13,958 DEBUG _L3_:323" + logStatement + "End workflow state change\n");
+ sb1.append("2009-06-24 02:43:13,958 DEBUG _L4_:323 ABC" + logStatement + "End workflow state change\n");
+ sb1.append("2009-06-24 02:43:13,958 DEBUG _L5_:323 !@#$%^&*() blah blah" + logStatement + "End workflow state change\n");
+ sb1.append("2009-06-24 02:43:13,958 WARN _L6_:323 -" + logStatement + "End workflow state change\n");
+ sb1.append("2009-06-24 02:43:13,958 WARN _L7_:323 +" + logStatement + "End workflow state change\n");
+ sb1.append("2009-06-24 02:43:13,958 WARN _L8_:323" + logStatement + "End workflow state change\n");
+ sb1.append("2009-06-24 02:43:13,958 WARN _L9_:323 ABC" + logStatement + "End workflow state change\n");
+ sb1.append("2009-06-24 02:43:13,958 WARN _L10_:323 !@#$%^&*() blah blah" + logStatement + "End workflow state change\n");
+ fw1.write(sb1.toString());
+ fw1.close();
+ File f1 = new File(getTestCaseDir() + "/oozie.log");
+ f1.setLastModified(currTime);
+
+ StringWriter sw = new StringWriter();
+ XLogStreamer str = new XLogStreamer(xf, sw, getTestCaseDir(), "oozie.log", 1);
+ str.streamLog(new Date(currTime - 5000), new Date(currTime + 5000));
+ String[] out = sw.toString().split("\n");
+ // Check if the retrieved log content is of length five lines after filtering; we expect the first five lines because the
+ // filtering shouldn't care whether or not there is a dash while the last five lines don't pass the normal filtering
+ assertEquals(5, out.length);
+ // Check if the lines of the log contain the expected strings
+ assertEquals(true, out[0].contains("_L1_"));
+ assertEquals(true, out[1].contains("_L2_"));
+ assertEquals(true, out[2].contains("_L3_"));
+ assertEquals(true, out[3].contains("_L4_"));
+ assertEquals(true, out[4].contains("_L5_"));
+ }
+
private void writeToGZFile(File f, StringBuilder sbr) throws IOException {
GZIPOutputStream gzout = new GZIPOutputStream(new FileOutputStream(f));
String strg = sbr.toString();
Added: oozie/branches/branch-3.3/core/src/test/resources/test-no-dash-log4j.properties
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/resources/test-no-dash-log4j.properties?rev=1442212&view=auto
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/resources/test-no-dash-log4j.properties (added)
+++ oozie/branches/branch-3.3/core/src/test/resources/test-no-dash-log4j.properties Mon Feb 4 17:57:13 2013
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+log4j.appender.oozie=org.apache.log4j.rolling.RollingFileAppender
+log4j.appender.oozie.RollingPolicy=org.apache.oozie.util.OozieRollingPolicy
+log4j.appender.oozie.File=${oozie.log.dir}/oozie.log
+log4j.appender.oozie.Append=true
+log4j.appender.oozie.layout=org.apache.log4j.PatternLayout
+# The default conversion pattern is %d{ISO8601} %5p %c{1}:%L - %m%n
+# We're testing if log streaming works without the dash
+log4j.appender.oozie.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L %m%n
+log4j.appender.oozie.RollingPolicy.FileNamePattern=${log4j.appender.oozie.File}-%d{yyyy-MM-dd-HH}
+log4j.appender.oozie.RollingPolicy.MaxHistory=720
+
+log4j.logger.a=ALL, oozie
Modified: oozie/branches/branch-3.3/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/release-log.txt?rev=1442212&r1=1442211&r2=1442212&view=diff
==============================================================================
--- oozie/branches/branch-3.3/release-log.txt (original)
+++ oozie/branches/branch-3.3/release-log.txt Mon Feb 4 17:57:13 2013
@@ -1,5 +1,6 @@
-- Oozie 3.3.2 (unreleased)
+OOZIE-999 XLogStreamer requires log4j.appender.oozie.layout.ConversionPattern to contain a "-" (dash) or streaming logs to web UI and CLI don't work (rkanter via virag)
OOZIE-998 test-patch doesn't allow lines that are exactly 132 characters (rkanter via tucu)
OOZIE-739 a coord action fails because the uri points to a namenode that is not in whitelist. the E0901 error shows in the oozie.log, but not written to the database (mona,mbattisha via virag)
OOZIE-987 Fix minor bug in one of the uber jar tests (rkanter via tucu)