You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pu...@apache.org on 2016/01/29 18:27:28 UTC
oozie git commit: OOZIE-2312 oozie doesn't purge audit and error log
Repository: oozie
Updated Branches:
refs/heads/master 05fce1f78 -> e94a345ac
OOZIE-2312 oozie doesn't purge audit and error log
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/e94a345a
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/e94a345a
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/e94a345a
Branch: refs/heads/master
Commit: e94a345acf220c04ee870515ad38adb9865c2720
Parents: 05fce1f
Author: Purshotam Shah <pu...@yahoo-inc.com>
Authored: Fri Jan 29 09:27:18 2016 -0800
Committer: Purshotam Shah <pu...@yahoo-inc.com>
Committed: Fri Jan 29 09:27:18 2016 -0800
----------------------------------------------------------------------
core/src/main/conf/oozie-log4j.properties | 7 +-
.../java/org/apache/oozie/service/XLogUtil.java | 5 +
.../apache/oozie/util/OozieRollingPolicy.java | 109 ++++++++++--------
.../oozie/util/TestOozieRollingPolicy.java | 112 ++++++++++---------
release-log.txt | 1 +
5 files changed, 130 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/e94a345a/core/src/main/conf/oozie-log4j.properties
----------------------------------------------------------------------
diff --git a/core/src/main/conf/oozie-log4j.properties b/core/src/main/conf/oozie-log4j.properties
index 2b20ff2..c563687 100644
--- a/core/src/main/conf/oozie-log4j.properties
+++ b/core/src/main/conf/oozie-log4j.properties
@@ -86,12 +86,15 @@ log4j.appender.oozieinstrumentation.Append=true
log4j.appender.oozieinstrumentation.layout=org.apache.log4j.PatternLayout
log4j.appender.oozieinstrumentation.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n
-log4j.appender.oozieaudit=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.oozieaudit.DatePattern='.'yyyy-MM-dd
+log4j.appender.oozieaudit=org.apache.log4j.rolling.RollingFileAppender
+log4j.appender.oozieaudit.RollingPolicy=org.apache.oozie.util.OozieRollingPolicy
log4j.appender.oozieaudit.File=${oozie.log.dir}/oozie-audit.log
log4j.appender.oozieaudit.Append=true
log4j.appender.oozieaudit.layout=org.apache.log4j.PatternLayout
log4j.appender.oozieaudit.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n
+log4j.appender.oozieaudit.RollingPolicy.FileNamePattern=${log4j.appender.oozieaudit.File}.%d{yyyy-MM-dd}
+log4j.appender.oozieaudit.RollingPolicy.MaxHistory=30
+
log4j.appender.openjpa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.openjpa.DatePattern='.'yyyy-MM-dd
http://git-wip-us.apache.org/repos/asf/oozie/blob/e94a345a/core/src/main/java/org/apache/oozie/service/XLogUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/XLogUtil.java b/core/src/main/java/org/apache/oozie/service/XLogUtil.java
index 59a4d22..aae00c2 100644
--- a/core/src/main/java/org/apache/oozie/service/XLogUtil.java
+++ b/core/src/main/java/org/apache/oozie/service/XLogUtil.java
@@ -96,6 +96,11 @@ public class XLogUtil {
if (pattern.matches(Pattern.quote(logFile) + ".*-%d\\{yyyy-MM-dd-HH\\}(\\.gz)?")) {
logRotation = 60 * 60;
}
+ else if (pattern.matches(Pattern.quote(logFile) + ".*-%d\\{yyyy-MM-dd\\}(\\.gz)?")
+ || pattern.matches(Pattern.quote(logFile) + ".*\\.%d\\{yyyy-MM-dd\\}(\\.gz)?")) {
+ logRotation = 60 * 60 * 24;
+ }
+
else {
log.warn(
"Oozie WS "
http://git-wip-us.apache.org/repos/asf/oozie/blob/e94a345a/core/src/main/java/org/apache/oozie/util/OozieRollingPolicy.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/OozieRollingPolicy.java b/core/src/main/java/org/apache/oozie/util/OozieRollingPolicy.java
index 19ce7f9..625053c 100644
--- a/core/src/main/java/org/apache/oozie/util/OozieRollingPolicy.java
+++ b/core/src/main/java/org/apache/oozie/util/OozieRollingPolicy.java
@@ -19,17 +19,21 @@
package org.apache.oozie.util;
import java.io.File;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Calendar;
import java.util.Collections;
+import java.util.List;
import java.util.concurrent.Semaphore;
-import java.util.regex.Matcher;
+
import org.apache.log4j.Appender;
+import org.apache.log4j.pattern.ExtrasPatternParser;
import org.apache.log4j.rolling.RollingPolicyBase;
import org.apache.log4j.rolling.RolloverDescription;
import org.apache.log4j.rolling.TimeBasedRollingPolicy;
import org.apache.log4j.rolling.TriggeringPolicy;
import org.apache.log4j.spi.LoggingEvent;
+import org.apache.log4j.pattern.LiteralPatternConverter;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.XLogService;
@@ -49,7 +53,10 @@ public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringP
private Thread deleteThread;
private int maxHistory = 720; // (720 hours / 24 hours per day = 30 days) as default
-
+
+ String oozieLogDir;
+ String logFileName;
+
public int getMaxHistory() {
return maxHistory;
}
@@ -63,7 +70,28 @@ public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringP
deleteThread = new Thread();
tbrp = new TimeBasedRollingPolicy();
}
-
+
+
+ @SuppressWarnings("rawtypes")
+ public void setFileNamePattern(String fnp) {
+ super.setFileNamePattern(fnp);
+ List converters = new ArrayList();
+ StringBuffer file = new StringBuffer();
+ ExtrasPatternParser
+ .parse(fnp, converters, new ArrayList(), null, ExtrasPatternParser.getFileNamePatternRules());
+ if (!converters.isEmpty()) {
+ ((LiteralPatternConverter) converters.get(0)).format(null, file);
+ File f = new File(file.toString());
+ oozieLogDir = f.getParent();
+ logFileName = f.getName();
+ }
+ else {
+ XLogService xls = Services.get().get(XLogService.class);
+ oozieLogDir = xls.getOozieLogPath();
+ logFileName = xls.getOozieLogName();
+ }
+
+ }
@Override
public void activateOptions() {
super.activateOptions();
@@ -106,34 +134,30 @@ public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringP
private void deleteOldFiles() {
ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
- XLogService xls = getXLogService();
- if (xls != null) { // We need this to get the paths
- String oozieLogPath = xls.getOozieLogPath();
- String logFile = xls.getOozieLogName();
- if (oozieLogPath != null && logFile != null) {
- String[] children = new File(oozieLogPath).list();
- if (children != null) {
- for (String child : children) {
- if (child.startsWith(logFile) && !child.equals(logFile)) {
- File childFile = new File(new File(oozieLogPath).getAbsolutePath(), child);
- if (child.endsWith(".gz")) {
- long gzFileCreationTime = getGZFileCreationTime(child);
- if (gzFileCreationTime != -1) {
- fileList.add(new FileInfo(childFile.getAbsolutePath(), gzFileCreationTime));
- }
- } else{
- long modTime = childFile.lastModified();
- fileList.add(new FileInfo(childFile.getAbsolutePath(), modTime));
+ if (oozieLogDir != null && logFileName != null) {
+ String[] children = new File(oozieLogDir).list();
+ if (children != null) {
+ for (String child : children) {
+ if (child.startsWith(logFileName) && !child.equals(logFileName)) {
+ File childFile = new File(new File(oozieLogDir).getAbsolutePath(), child);
+ if (child.endsWith(".gz")) {
+ long gzFileCreationTime = getGZFileCreationTime(child);
+ if (gzFileCreationTime != -1) {
+ fileList.add(new FileInfo(childFile.getAbsolutePath(), gzFileCreationTime));
}
}
+ else {
+ long modTime = childFile.lastModified();
+ fileList.add(new FileInfo(childFile.getAbsolutePath(), modTime));
+ }
}
}
}
}
-
+
if (fileList.size() > maxHistory) {
Collections.sort(fileList);
-
+
for (int i = maxHistory; i < fileList.size(); i++) {
new File(fileList.get(i).getFileName()).delete();
}
@@ -141,23 +165,22 @@ public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringP
}
private long getGZFileCreationTime(String fileName) {
- // Default return value of -1 to exclude the file
- long returnVal = -1;
- Matcher m = XLogStreamer.gzTimePattern.matcher(fileName);
- if (m.matches() && m.groupCount() == 4) {
- int year = Integer.parseInt(m.group(1));
- int month = Integer.parseInt(m.group(2));
- int day = Integer.parseInt(m.group(3));
- int hour = Integer.parseInt(m.group(4));
- int minute = 0;
- Calendar calendarEntry = Calendar.getInstance();
- calendarEntry.set(year, month - 1, day, hour, minute); // give month-1(Say, 7 for August)
- long logFileStartTime = calendarEntry.getTimeInMillis();
- returnVal = logFileStartTime;
+ SimpleDateFormat formatter;
+ String date = fileName.substring(logFileName.length(), fileName.length() - 3); //3 for .gz
+ if (date.length() == 10) {
+ formatter = new SimpleDateFormat("yyyy-MM-dd");
+ }
+ else {
+ formatter = new SimpleDateFormat("yyyy-MM-dd-HH");
+ }
+ try {
+ return formatter.parse(date).getTime();
+ }
+ catch (ParseException e) {
+ return -1;
}
- return returnVal;
}
-
+
class FileInfo implements Comparable<FileInfo> {
String fileName;
long modTime;
@@ -190,12 +213,4 @@ public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringP
}
}
- // Needed for TestOozieRollingPolicy tests to be able to override getOozieLogPath() and getOozieLogName()
- // by overriding getXLogService()
- XLogService getXLogService() {
- if (Services.get() != null) {
- return Services.get().get(XLogService.class);
- }
- return null;
- }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/e94a345a/core/src/test/java/org/apache/oozie/util/TestOozieRollingPolicy.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/util/TestOozieRollingPolicy.java b/core/src/test/java/org/apache/oozie/util/TestOozieRollingPolicy.java
index 9ae7bec..051ce56 100644
--- a/core/src/test/java/org/apache/oozie/util/TestOozieRollingPolicy.java
+++ b/core/src/test/java/org/apache/oozie/util/TestOozieRollingPolicy.java
@@ -19,11 +19,12 @@
package org.apache.oozie.util;
import java.io.File;
+import java.io.IOException;
import java.util.Calendar;
import java.util.GregorianCalendar;
+
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.LogManager;
-import org.apache.oozie.service.XLogService;
import org.apache.oozie.test.XTestCase;
public class TestOozieRollingPolicy extends XTestCase {
@@ -41,40 +42,56 @@ public class TestOozieRollingPolicy extends XTestCase {
LogManager.resetConfiguration();
super.tearDown();
}
-
+
public void testDeletingOldFiles() throws Exception {
+ _testDeletingOldFiles("oozie.log");
+ }
+
+ public void testDeletingErrorOldFiles() throws Exception {
+ _testDeletingOldFiles("oozie-error.log");
+ }
+
+ public void testDeletingAuditOldFiles() throws Exception {
+ _testDeletingOldFiles("oozie-audit.log", Calendar.DAY_OF_MONTH);
+ }
+
+ private void _testDeletingOldFiles(String oozieLogName) throws IOException {
+ _testDeletingOldFiles(oozieLogName, Calendar.HOUR_OF_DAY);
+ }
+
+ private void _testDeletingOldFiles(String oozieLogName, int calendarUnit) throws IOException{
// OozieRollingPolicy gets the log path and log name from XLogService by calling Services.get.get(XLogService.class) so we
// use a mock version where we overwrite the XLogService.getOozieLogName() and XLogService.getOozieLogPath() to simply
// return these values instead of involving Services. We then overwrite OozieRollingPolicy.getXLogService() to return the
// mock one instead.
String oozieLogPath = getTestCaseDir();
- String oozieLogName = "oozie.log";
- final XLogService xls = new MockXLogService(oozieLogPath, oozieLogName);
- OozieRollingPolicy orp = new OozieRollingPolicy() {
- @Override
- XLogService getXLogService() {
- return xls;
- }
-
- };
+ OozieRollingPolicy orp = new OozieRollingPolicy();
+
+ if (calendarUnit == Calendar.DAY_OF_MONTH) {
+ orp.setFileNamePattern(oozieLogPath + "/" + oozieLogName + "-%d{yyyy-MM-dd}");
+ }
+ else {
+ orp.setFileNamePattern(oozieLogPath + "/" + oozieLogName + "-%d{yyyy-MM-dd-HH}");
+
+ }
orp.setMaxHistory(3); // only keep 3 newest logs
Calendar cal = new GregorianCalendar();
final File f0 = new File(oozieLogPath, oozieLogName);
f0.createNewFile();
f0.setLastModified(cal.getTimeInMillis());
- cal.add(Calendar.HOUR_OF_DAY, 1);
- final File f1 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal) + ".gz");
+ cal.add(calendarUnit, 1);
+ final File f1 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal, calendarUnit) + ".gz");
f1.createNewFile();
- cal.add(Calendar.HOUR_OF_DAY, 1);
- final File f2 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal) + ".gz");
+ cal.add(calendarUnit, 1);
+ final File f2 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal, calendarUnit) + ".gz");
f2.createNewFile();
- cal.add(Calendar.HOUR_OF_DAY, 1);
- final File f3 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal) + ".gz");
+ cal.add(calendarUnit, 1);
+ final File f3 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal, calendarUnit) + ".gz");
f3.createNewFile();
- cal.add(Calendar.HOUR_OF_DAY, 1);
- final File f4 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal) + ".gz");
+ cal.add(calendarUnit, 1);
+ final File f4 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal, calendarUnit) + ".gz");
f4.createNewFile();
// Test that it only deletes the oldest file (f1)
@@ -87,13 +104,13 @@ public class TestOozieRollingPolicy extends XTestCase {
});
assertTrue(f0.exists() && !f1.exists() && f2.exists() && f3.exists() && f4.exists());
- cal.add(Calendar.HOUR_OF_DAY, 1);
- final File f5 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal));
+ cal.add(calendarUnit, 1);
+ final File f5 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal, calendarUnit));
f5.createNewFile();
f5.setLastModified(cal.getTimeInMillis());
- cal.add(Calendar.HOUR_OF_DAY, -15);
- final File f6 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal));
+ cal.add(calendarUnit, -15);
+ final File f6 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal, calendarUnit));
f6.createNewFile();
f6.setLastModified(cal.getTimeInMillis());
@@ -111,9 +128,9 @@ public class TestOozieRollingPolicy extends XTestCase {
f7.createNewFile();
f7.setLastModified(cal.getTimeInMillis());
- cal.add(Calendar.HOUR_OF_DAY, 1);
- final File f8 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal));
- cal.add(Calendar.HOUR_OF_DAY, 15);
+ cal.add(calendarUnit, 1);
+ final File f8 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal, calendarUnit));
+ cal.add(calendarUnit, 15);
f8.createNewFile();
f8.setLastModified(cal.getTimeInMillis());
@@ -130,19 +147,22 @@ public class TestOozieRollingPolicy extends XTestCase {
assertTrue(f0.exists() && !f1.exists() && !f2.exists() && !f3.exists() && f4.exists() && f5.exists() && !f6.exists() &&
f7.exists() && f8.exists());
}
+
- private String formatDateForFilename(Calendar cal) {
+ private String formatDateForFilename(Calendar cal, int calendarUnit) {
int year = cal.get(Calendar.YEAR);
int month = cal.get(Calendar.MONTH) + 1;
int date = cal.get(Calendar.DATE);
int hour = cal.get(Calendar.HOUR_OF_DAY);
-
+
StringBuilder sb = new StringBuilder("-");
if (year < 10) {
sb.append("000");
- } else if (year < 100) {
+ }
+ else if (year < 100) {
sb.append("00");
- } else if (year < 1000) {
+ }
+ else if (year < 1000) {
sb.append("0");
}
sb.append(year);
@@ -156,33 +176,15 @@ public class TestOozieRollingPolicy extends XTestCase {
sb.append("0");
}
sb.append(date);
- sb.append("-");
- if (hour < 10) {
- sb.append("0");
+ if (calendarUnit == Calendar.HOUR_OF_DAY) {
+ sb.append("-");
+ if (hour < 10) {
+ sb.append("0");
+ }
+ sb.append(hour);
+
}
- sb.append(hour);
-
return sb.toString();
}
-
- class MockXLogService extends XLogService {
-
- private String logPath;
- private String logName;
-
- public MockXLogService(String lp, String ln) {
- logPath = lp;
- logName = ln;
- }
-
- @Override
- public String getOozieLogName() {
- return logName;
- }
- @Override
- public String getOozieLogPath() {
- return logPath;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/e94a345a/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index e3925de..7e9c01b 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2312 oozie doesn't purge audit and error log (puru)
OOZIE-2431 Support hive.metastore.uris as alias for hcat.metastore.uri (satishsaley via puru)
OOZIE-1976 Specifying coordinator input datasets in more logical ways (puru)
OOZIE-2444 Need conditional logic in bundles (satishsaley via puru)