You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@chukwa.apache.org by as...@apache.org on 2009/07/11 00:36:03 UTC
svn commit: r793128 - in /hadoop/chukwa/trunk: ./
src/java/org/apache/hadoop/chukwa/datacollection/adaptor/
src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/
src/java/org/apache/hadoop/chukwa/datacollection/agent/
src/java/org/apache...
Author: asrabkin
Date: Fri Jul 10 22:36:02 2009
New Revision: 793128
URL: http://svn.apache.org/viewvc?rev=793128&view=rev
Log:
CHUKWA-284. Checkpoint includes last byte Acked, not sent
Modified:
hadoop/chukwa/trunk/CHANGES.txt
hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/Adaptor.java
hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/DirTailingAdaptor.java
hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/ExecAdaptor.java
hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java
hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java
hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/MaxRateSender.java
hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestExecAdaptor.java
hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestFileAdaptor.java
Modified: hadoop/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/CHANGES.txt?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/CHANGES.txt (original)
+++ hadoop/chukwa/trunk/CHANGES.txt Fri Jul 10 22:36:02 2009
@@ -34,6 +34,8 @@
IMPROVEMENTS
+ CHUKWA-284. Checkpoint includes last byte Acked, not sent. (asrabkin)
+
CHUKWA-25. Archiver can group by cluster name. (asrabkin)
CHUKWA-308. Added capability to start hicc without tomcat. (Eric Yang)
Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/Adaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/Adaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/Adaptor.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/Adaptor.java Fri Jul 10 22:36:02 2009
@@ -51,8 +51,9 @@
ChunkReceiver dest, AdaptorManager c) throws AdaptorException;
/**
- * Return the adaptor's state Should not include class name, datatype or byte
- * offset, which are written by caller.
+ * Return the adaptor's state Should not include class name or byte
+ * offset, which are written by caller. The datatype should, however,
+ * be written by this method.
*
* @return the adaptor state as a string
* @throws AdaptorException
Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/DirTailingAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/DirTailingAdaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/DirTailingAdaptor.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/DirTailingAdaptor.java Fri Jul 10 22:36:02 2009
@@ -92,7 +92,7 @@
@Override
public String getCurrentStatus() throws AdaptorException {
try {
- return type + " " + baseDir.getCanonicalPath()+ " " + adaptorName+ " " + lastSweepStartTime;
+ return type + " " + baseDir.getCanonicalPath()+ " " + adaptorName;
} catch(IOException e) {
throw new AdaptorException(e);
}
Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/ExecAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/ExecAdaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/ExecAdaptor.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/ExecAdaptor.java Fri Jul 10 22:36:02 2009
@@ -123,7 +123,7 @@
@Override
public String getCurrentStatus() throws AdaptorException {
- return type + " " + period + " " + cmd + " " + sendOffset;
+ return type + " " + period + " " + cmd;
}
Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java Fri Jul 10 22:36:02 2009
@@ -313,8 +313,7 @@
@Override
public String getCurrentStatus() throws AdaptorException {
- return type.trim() + " " + offsetOfFirstByte + " " + toWatch.getPath()
- + " " + fileReadOffset;
+ return type.trim() + " " + offsetOfFirstByte + " " + toWatch.getPath();
}
}
Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java Fri Jul 10 22:36:02 2009
@@ -77,7 +77,6 @@
public void start(long bytes) {
this.attempts = 0;
-
log.info("started file tailer on file " + toWatch
+ " with first byte at offset " + offsetOfFirstByte);
@@ -177,8 +176,7 @@
* @see org.apache.hadoop.chukwa.datacollection.adaptor.Adaptor#getCurrentStatus()
*/
public String getCurrentStatus() {
- return type.trim() + " " + offsetOfFirstByte + " " + toWatch.getPath()
- + " " + fileReadOffset;
+ return type.trim() + " " + offsetOfFirstByte + " " + toWatch.getPath();
// can make this more efficient using a StringBuilder
}
Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java Fri Jul 10 22:36:02 2009
@@ -445,7 +445,7 @@
new OutputStreamWriter(fos)));
for (Map.Entry<String, String> stat : getAdaptorList().entrySet()) {
- out.println("ADD "+ stat.getKey()+ " = " + stat.getValue());
+ out.println("ADD "+ stat.getKey()+ " = " + stat.getValue());
}
out.close();
@@ -490,7 +490,8 @@
private String formatAdaptorStatus(Adaptor a) throws AdaptorException {
- return a.getClass().getCanonicalName() + " " + a.getCurrentStatus();
+ return a.getClass().getCanonicalName() + " " + a.getCurrentStatus() +
+ " " + adaptorPositions.get(a).offset;
}
/**
Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java Fri Jul 10 22:36:02 2009
@@ -52,7 +52,7 @@
String adaptorName;
String type;
String params;
- long offset;
+ private long offset;
long numRetries;
long retryInterval;
Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java Fri Jul 10 22:36:02 2009
@@ -40,7 +40,7 @@
private volatile boolean stopping = false;
public String getCurrentStatus() throws AdaptorException {
- return type.trim() + " " + bytesPerSec + " " + offset;
+ return type.trim() + " " + bytesPerSec;
}
public void start(String adaptorID, String type,
Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/MaxRateSender.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/MaxRateSender.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/MaxRateSender.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/MaxRateSender.java Fri Jul 10 22:36:02 2009
@@ -37,7 +37,7 @@
private String adaptorID;
public String getCurrentStatus() throws AdaptorException {
- return "";
+ return type;
}
public void start(String adaptorID, String type, long offset,
Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestExecAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestExecAdaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestExecAdaptor.java (original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestExecAdaptor.java Fri Jul 10 22:36:02 2009
@@ -45,7 +45,10 @@
agent.shutdown();
}
-
+ /*
+ * Buzz in a loop, starting ls every 100 ms.
+ * Length of loop controlled by sleep statement near bottom of function
+ */
public void testForLeaks() throws ChukwaAgent.AlreadyRunningException, InterruptedException {
Configuration conf = new Configuration();
// conf.set("chukwaAgent.control.port", "0");
@@ -57,7 +60,7 @@
assertEquals(0, agent.adaptorCount());
String lsID = agent.processAddCommand(
"add exec= org.apache.hadoop.chukwa.datacollection.adaptor.ExecAdaptor Listing 100 /bin/sleep 1 0");
- Thread.sleep( 60*1000);
+ Thread.sleep( 25*1000); //RAISE THIS to test longer
System.out.println("stopped ok");
}
Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestFileAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestFileAdaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestFileAdaptor.java (original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestFileAdaptor.java Fri Jul 10 22:36:02 2009
@@ -84,7 +84,7 @@
public void testRepeatedly() throws IOException,
ChukwaAgent.AlreadyRunningException, InterruptedException {
- int tests = 300; //SHOULD SET HIGHER AND WATCH WITH lsof to find leaks
+ int tests = 100; //SHOULD SET HIGHER AND WATCH WITH lsof to find leaks
ChukwaAgent agent = new ChukwaAgent(conf);
for(int i=0; i < tests; ++i) {