You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@chukwa.apache.org by ey...@apache.org on 2009/06/03 23:34:56 UTC
svn commit: r781590 - in /hadoop/chukwa/trunk: ./ conf/
src/java/org/apache/hadoop/chukwa/datacollection/controller/
src/java/org/apache/hadoop/chukwa/inputtools/log4j/
src/test/org/apache/hadoop/chukwa/inputtools/log4j/
Author: eyang
Date: Wed Jun 3 21:34:54 2009
New Revision: 781590
URL: http://svn.apache.org/viewvc?rev=781590&view=rev
Log:
CHUKWA-267. Added TaskLogAppender for collecting Task Tracker Log file. (Eric Yang)
Added:
hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ClientFinalizer.java
hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/TaskLogAppender.java
hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestTaskLogAppender.java
hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/tasklog-log4j.properties
Modified:
hadoop/chukwa/trunk/build.xml
hadoop/chukwa/trunk/conf/hadoop-log4j.properties
hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
Modified: hadoop/chukwa/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/build.xml?rev=781590&r1=781589&r2=781590&view=diff
==============================================================================
--- hadoop/chukwa/trunk/build.xml (original)
+++ hadoop/chukwa/trunk/build.xml Wed Jun 3 21:34:54 2009
@@ -449,6 +449,7 @@
<copy todir="${test.build.dir}/classes/org/apache/hadoop/chukwa/inputtools/log4j/">
<fileset dir="${basedir}/src/test/org/apache/hadoop/chukwa/inputtools/log4j/">
<include name="late-log4j.properties" />
+ <include name="tasklog-log4j.properties" />
</fileset>
</copy>
<copy todir="${test.build.dir}/classes">
@@ -477,6 +478,7 @@
<env key="CHUKWA_CONF_DIR" value="${test.build.dir}/conf" />
<env key="CHUKWA_DATA_DIR" value="${test.build.dir}/var" />
<sysproperty key="test.src.dir" value="${test.src.dir}" />
+ <sysproperty key="test.build.classes" value="${test.build.classes}" />
<formatter type="${test.junit.output.format}" />
<batchtest todir="${test.build.dir}" unless="testcase">
<fileset dir="${test.src.dir}">
Modified: hadoop/chukwa/trunk/conf/hadoop-log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/hadoop-log4j.properties?rev=781590&r1=781589&r2=781590&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/hadoop-log4j.properties (original)
+++ hadoop/chukwa/trunk/conf/hadoop-log4j.properties Wed Jun 3 21:34:54 2009
@@ -99,7 +99,8 @@
hadoop.tasklog.purgeLogSplits=true
hadoop.tasklog.logsRetainHours=12
-log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
+log4j.appender.TLA=org.apache.hadoop.chukwa.inputtools.log4j.TaskLogAppender
+log4j.appender.TLA.recordType=TaskLog
log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java?rev=781590&r1=781589&r2=781590&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java Wed Jun 3 21:34:54 2009
@@ -186,6 +186,7 @@
}
Map<Long, ChukwaAgentController.Adaptor> runningAdaptors = new HashMap<Long, Adaptor>();
+ Map<Long, ChukwaAgentController.Adaptor> runningInstanceAdaptors = new HashMap<Long, Adaptor>();
Map<Long, ChukwaAgentController.Adaptor> pausedAdaptors;
String hostname;
int portno;
@@ -259,6 +260,7 @@
if (adaptorID > 0) {
runningAdaptors.put(adaptorID, adaptor);
+ runningInstanceAdaptors.put(adaptorID, adaptor);
} else {
System.err
.println("Failed to successfully add the adaptor in AgentClient, adaptorID returned by add() was negative.");
@@ -323,6 +325,21 @@
}
}
+ public void removeInstanceAdaptors() {
+ // Remove adaptors created by this instance of chukwa agent controller.
+ // Instead of removing using id, this is removed by using the stream name
+ // and record type. This prevents the system to shutdown the wrong
+ // adaptor after agent crashes.
+ for (Adaptor a : runningInstanceAdaptors.values()) {
+ try {
+ remove(a.name, a.appType, a.params);
+ } catch (IOException ioe) {
+ System.out.println("Error removing an adaptor in removeInstanceAdaptors()");
+ ioe.printStackTrace();
+ }
+ }
+ }
+
Map<Long, ChukwaAgentController.Adaptor> list() throws IOException {
Socket s = new Socket(hostname, portno);
try {
Added: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ClientFinalizer.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ClientFinalizer.java?rev=781590&view=auto
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ClientFinalizer.java (added)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ClientFinalizer.java Wed Jun 3 21:34:54 2009
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.chukwa.datacollection.controller;
+
+import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
+import org.apache.log4j.Logger;
+import org.apache.log4j.helpers.LogLog;
+
+
+public class ClientFinalizer extends Thread {
+ private ChukwaAgentController chukwaClient = null;
+ private String recordType = null;
+ private String fileName = null;
+
+ public ClientFinalizer(ChukwaAgentController chukwaClient) {
+ this.chukwaClient = chukwaClient;
+ }
+
+ public synchronized void run() {
+ try {
+ if (chukwaClient != null) {
+ chukwaClient.removeInstanceAdaptors();
+ } else {
+ LogLog.warn("chukwaClient is null cannot do any cleanup");
+ }
+ } catch (Throwable e) {
+ LogLog.warn("closing the controller threw an exception:\n" + e);
+ }
+ }
+}
+
Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java?rev=781590&r1=781589&r2=781590&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java Wed Jun 3 21:34:54 2009
@@ -23,6 +23,7 @@
import java.util.regex.Pattern;
import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
+import org.apache.hadoop.chukwa.datacollection.controller.ClientFinalizer;
import org.apache.hadoop.chukwa.util.RecordConstants;
import org.apache.log4j.FileAppender;
import org.apache.log4j.Layout;
@@ -460,33 +461,6 @@
}
}
- private class ClientFinalizer extends Thread {
- private ChukwaAgentController chukwaClient = null;
- private String recordType = null;
- private String fileName = null;
-
- public ClientFinalizer(ChukwaAgentController chukwaClient,
- String recordType, String fileName) {
- this.chukwaClient = chukwaClient;
- this.recordType = recordType;
- this.fileName = fileName;
- }
-
- public synchronized void run() {
- try {
- if (chukwaClient != null) {
- log.debug("ShutdownHook: removing:" + fileName);
- chukwaClient.removeFile(recordType, fileName);
- } else {
- LogLog.warn("chukwaClient is null cannot do any cleanup");
- }
- } catch (Throwable e) {
- LogLog.warn("closing the controller threw an exception:\n" + e);
- }
- }
- }
-
-
/**
* Fix for late-initialisation
*/
@@ -573,8 +547,7 @@
numRetries, retryInterval);
// Setup a shutdownHook for the controller
- clientFinalizer = new ClientFinalizer(chukwaClient, recordType,
- log4jFileName);
+ clientFinalizer = new ClientFinalizer(chukwaClient);
Runtime.getRuntime().addShutdownHook(clientFinalizer);
if (adaptorID > 0) {
Added: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/TaskLogAppender.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/TaskLogAppender.java?rev=781590&view=auto
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/TaskLogAppender.java (added)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/TaskLogAppender.java Wed Jun 3 21:34:54 2009
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.chukwa.inputtools.log4j;
+
+import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
+import org.apache.hadoop.chukwa.datacollection.controller.ClientFinalizer;
+import org.apache.hadoop.chukwa.util.RecordConstants;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+
+public class TaskLogAppender extends
+ org.apache.hadoop.mapred.TaskLogAppender {
+ static Logger log = Logger.getLogger(TaskLogAppender.class);
+ static final String adaptorType = ChukwaAgentController.CharFileTailUTF8NewLineEscaped;
+ ChukwaAgentController chukwaClient;
+ String recordType = null;
+ static boolean chukwaClientIsNull = true;
+ static final Object chukwaLock = new Object();
+ private ClientFinalizer clientFinalizer = null;
+
+ public String getRecordType() {
+ if (recordType != null)
+ return recordType;
+ else
+ return "unknown";
+ }
+
+ public void setRecordType(String recordType) {
+ this.recordType = recordType;
+ }
+
+ public void subAppend(LoggingEvent event) {
+ this.qw.write(RecordConstants.escapeAllButLastRecordSeparator("\n",this.layout.format(event)));
+ // Make sure only one thread can do this
+ // and use the boolean to avoid the first level locking
+ if (chukwaClientIsNull) {
+ synchronized (chukwaLock) {
+ if (chukwaClient == null) {
+ String log4jFileName = getFile();
+ String recordType = getRecordType();
+ long currentLength = 0L;
+ chukwaClient = new ChukwaAgentController();
+ chukwaClientIsNull = false;
+ long adaptorID = chukwaClient.add(ChukwaAgentController.CharFileTailUTF8NewLineEscaped,
+ recordType,currentLength + " " + log4jFileName, currentLength);
+
+ // Setup a shutdownHook for the controller
+ clientFinalizer = new ClientFinalizer(chukwaClient);
+ Runtime.getRuntime().addShutdownHook(clientFinalizer);
+ if (adaptorID > 0) {
+ log.debug("Added file tailing adaptor to chukwa agent for file "
+ + log4jFileName + ", adaptorId:" + adaptorID
+ + " using this recordType :" + recordType
+ + ", starting at offset:" + currentLength);
+ } else {
+ log.debug("Chukwa adaptor not added, addFile(" + log4jFileName
+ + ") returned " + adaptorID
+ + ", current offset:" + currentLength);
+ }
+ }
+ }
+ }
+ }
+}
Added: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestTaskLogAppender.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestTaskLogAppender.java?rev=781590&view=auto
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestTaskLogAppender.java (added)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestTaskLogAppender.java Wed Jun 3 21:34:54 2009
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.chukwa.inputtools.log4j;
+
+import java.io.File;
+import java.net.URL;
+import java.util.Date;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.log4j.helpers.LogLog;
+import org.apache.log4j.helpers.OptionConverter;
+
+public class TestTaskLogAppender extends TestCase {
+
+ @SuppressWarnings("deprecation")
+ public void testTaskLogAppender() {
+
+ String folder = System.getProperty("test.build.classes");
+ File logFile = new File(folder + "/userlogs/attempt_200905220200_13470_r_000000_0/syslog");
+ if (logFile.exists()) {
+ logFile.delete();
+ }
+ Assert.assertTrue("Log file should not be there", logFile.exists() == false);
+ File tempDir = new File(folder);
+ if (!tempDir.exists()) {
+ tempDir.mkdirs();
+ }
+ // load new log4j
+ String configuratorClassName = OptionConverter.getSystemProperty(
+ LogManager.CONFIGURATOR_CLASS_KEY, null);
+
+ URL url = TestTaskLogAppender.class
+ .getResource("tasklog-log4j.properties");
+
+ System.getProperties().setProperty("CHUKWA_LOG_DIR", folder);
+
+ if (url != null) {
+ LogLog
+ .debug("Using URL [" + url + "] for automatic log4j configuration.");
+ try {
+ OptionConverter.selectAndConfigure(url, configuratorClassName,
+ LogManager.getLoggerRepository());
+ } catch (NoClassDefFoundError e) {
+ LogLog.warn("Error during default initialization", e);
+ }
+ } else {
+ Assert.fail("URL should not be null");
+ }
+ Logger log = Logger.getLogger(TestTaskLogAppender.class);
+ try {
+ Thread.sleep(2000);
+ }catch (Exception e) {
+ // do nothing
+ }
+ log.warn("test 123 " + new Date());
+ Assert.assertTrue("Log file should exist", logFile.exists() == true);
+ logFile.delete();
+ }
+
+}
Added: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/tasklog-log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/tasklog-log4j.properties?rev=781590&view=auto
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/tasklog-log4j.properties (added)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/tasklog-log4j.properties Wed Jun 3 21:34:54 2009
@@ -0,0 +1,11 @@
+log4j.rootLogger=INFO, R
+log4j.appender.R=org.apache.hadoop.chukwa.inputtools.log4j.TaskLogAppender
+log4j.appender.R.recordType=TaskLog
+log4j.appender.R.taskId=attempt_200905220200_13470_r_000000_0
+log4j.appender.R.totalLogFileSize=5000
+log4j.appender.R.layout=org.apache.log4j.PatternLayout
+log4j.appender.R.layout.ConversionPattern=%d{ISO8601} %p %t %c{1} - %m%n
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p %t %c{1} - %m%n