You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2009/02/24 22:12:33 UTC
svn commit: r747547 - in /hadoop/core/trunk: CHANGES.txt
src/contrib/chukwa/conf/system-data-loader.properties
src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
Author: cdouglas
Date: Tue Feb 24 21:12:33 2009
New Revision: 747547
URL: http://svn.apache.org/viewvc?rev=747547&view=rev
Log:
HADOOP-5042. Add a cleanup stage to log rollover in Chukwa appender. Contributed by Jerome Boulon.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/contrib/chukwa/conf/system-data-loader.properties
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=747547&r1=747546&r2=747547&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Feb 24 21:12:33 2009
@@ -131,6 +131,9 @@
HADOOP-4191. Adds a testcase for JobHistory. (Ravi Gummadi via ddas)
+ HADOOP-5042. Add a cleanup stage to log rollover in Chukwa appender.
+ (Jerome Boulon via cdouglas)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/core/trunk/src/contrib/chukwa/conf/system-data-loader.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/system-data-loader.properties?rev=747547&r1=747546&r2=747547&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/system-data-loader.properties (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/system-data-loader.properties Tue Feb 24 21:12:33 2009
@@ -2,6 +2,8 @@
log4j.appender.R=org.apache.hadoop.chukwa.inputtools.log4j.ChukwaDailyRollingFileAppender
log4j.appender.R.File=${CHUKWA_LOG_DIR}/${RECORD_TYPE}.log
log4j.appender.R.recordType=${RECORD_TYPE}
+log4j.appender.R.maxBackupIndex=3
+log4j.appender.R.cleanUpRegex=$fileName.20*
log4j.appender.R.layout=org.apache.log4j.PatternLayout
log4j.appender.R.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java?rev=747547&r1=747546&r2=747547&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java (original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java Tue Feb 24 21:12:33 2009
@@ -10,13 +10,18 @@
package org.apache.hadoop.chukwa.inputtools.log4j;
import java.io.File;
+import java.io.FilenameFilter;
import java.io.IOException;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Calendar;
+import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
+import java.util.List;
import java.util.Locale;
import java.util.TimeZone;
+import java.util.regex.Pattern;
import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
import org.apache.hadoop.chukwa.util.RecordConstants;
@@ -130,8 +135,7 @@
<b>DatePattern</b> option. The text before the colon is interpeted
as the protocol specificaion of a URL which is probably not what
you want. */
-
-
+
public class ChukwaDailyRollingFileAppender extends FileAppender {
static Logger log = Logger.getLogger(ChukwaDailyRollingFileAppender.class);
@@ -172,6 +176,16 @@
The next time we estimate a rollover should occur. */
private long nextCheck = System.currentTimeMillis () - 1;
+ /**
+ * Regex to select log files to be deleted
+ */
+ private String cleanUpRegex = null;
+
+ /**
+ * Set the maximum number of backup files to keep around.
+ */
+ private int maxBackupIndex = 10;
+
Date now = new Date();
SimpleDateFormat sdf;
@@ -351,6 +365,7 @@
}
File file = new File(fileName);
+
boolean result = file.renameTo(target);
if(result) {
LogLog.debug(fileName +" -> "+ scheduledFilename);
@@ -367,8 +382,82 @@
errorHandler.error("setFile("+fileName+", false) call failed.");
}
scheduledFilename = datedFilename;
+ cleanUp();
+ }
+
+
+ public String getCleanUpRegex() {
+ return cleanUpRegex;
+ }
+
+ public void setCleanUpRegex(String cleanUpRegex) {
+ this.cleanUpRegex = cleanUpRegex;
+ }
+
+ public int getMaxBackupIndex() {
+ return maxBackupIndex;
+ }
+
+ public void setMaxBackupIndex(int maxBackupIndex) {
+ this.maxBackupIndex = maxBackupIndex;
}
+ protected synchronized void cleanUp() {
+ String regex = "";
+ try {
+ File actualFile = new File(fileName);
+
+ String directoryName = actualFile.getParent();
+ String actualFileName = actualFile.getName();
+ File dirList = new File(directoryName);
+
+
+ if (cleanUpRegex == null || !cleanUpRegex.contains("$fileName")) {
+ LogLog.error("cleanUpRegex == null || !cleanUpRegex.contains(\"$fileName\")");
+ return;
+ }
+ regex =cleanUpRegex.replace("$fileName", actualFileName);
+ String[] dirFiles = dirList.list(new LogFilter(actualFileName,regex));
+
+ List<String> files = new ArrayList<String>();
+ for(String file: dirFiles) {
+ files.add(file);
+ }
+ Collections.sort(files);
+
+ while(files.size() > maxBackupIndex) {
+ String file = files.remove(0);
+ File f = new File(directoryName + "/" +file);
+ f.delete();
+ LogLog.debug("Removing: " +file);
+ }
+ } catch(Exception e) {
+ errorHandler.error("cleanUp("+fileName+"," + regex +") call failed.");
+ }
+ }
+
+ private class LogFilter implements FilenameFilter {
+ private Pattern p = null;
+ private String logFile = null;
+
+ public LogFilter(String logFile,String regex) {
+ this.logFile = logFile;
+ p = Pattern.compile(regex);
+ }
+
+ @Override
+ public boolean accept(File dir, String name) {
+ // ignore current log file
+ if (name.intern() == this.logFile.intern() ) {
+ return false;
+ }
+ //ignore file without the same prefix
+ if (!name.startsWith(logFile)) {
+ return false;
+ }
+ return p.matcher(name).find();
+ }
+ }
private class ClientFinalizer extends Thread
{
@@ -614,3 +703,4 @@
return getTime();
}
}
+