You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ac...@apache.org on 2009/02/03 08:06:24 UTC
svn commit: r740237 - in /hadoop/core/trunk/src/contrib/chukwa: ./
src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/
src/java/org/apache/hadoop/chukwa/datacollection/agent/
src/java/org/apache/hadoop/chukwa/util/ src/test/org/apache/...
Author: acmurthy
Date: Tue Feb 3 07:06:24 2009
New Revision: 740237
URL: http://svn.apache.org/viewvc?rev=740237&view=rev
Log:
HADOOP-5138. Fixing failing test cases by separating TestCharFileTailing and TestFileTailing adaptors. Contributed by Jerome Boulon.
Added:
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestCharFileTailingAdaptorUTF8.java
Modified:
hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java
Modified: hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt?rev=740237&r1=740236&r2=740237&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt (original)
+++ hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt Tue Feb 3 07:06:24 2009
@@ -10,6 +10,9 @@
BUG FIXES
+ HADOOP-5138. Fixing failing test cases by separating TestCharFileTailing
+ and TestFileTailing adaptors. (Jerome Boulon via acmurthy)
+
Release 0.19.0 - Unreleased
IMPROVEMENTS
Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java?rev=740237&r1=740236&r2=740237&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java (original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java Tue Feb 3 07:06:24 2009
@@ -84,16 +84,15 @@
this.dest = dest;
this.attempts = 0;
- Pattern cmd = Pattern.compile("(\\d+)\\s+(.+)");
+ Pattern cmd = Pattern.compile("(\\d+)\\s+(.+)\\s");
Matcher m = cmd.matcher(params);
if(m.matches()) {
offsetOfFirstByte = Long.parseLong(m.group(1));
toWatch = new File(m.group(2));
} else {
- toWatch = new File(params);
+ toWatch = new File(params.trim());
}
-
this.fileReadOffset= bytes;
tailer.startWatchingFile(this);
}
Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java?rev=740237&r1=740236&r2=740237&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java (original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java Tue Feb 3 07:06:24 2009
@@ -267,7 +267,7 @@
// but can be arbitrarily many space
// delimited agent specific params )
// 4) offset
- Pattern addCmdPattern = Pattern.compile("[aA][dD][dD]\\s+(\\S+)\\s+(\\S+)\\s+(.*\\S)?\\s*(\\d+)\\s*");
+ Pattern addCmdPattern = Pattern.compile("[aA][dD][dD]\\s+(\\S+)\\s+(\\S+)\\s+(.*\\s)?\\s*(\\d+)\\s*");
// FIXME: should handle bad lines here
public long processCommand(String cmd)
{
Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java?rev=740237&r1=740236&r2=740237&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java (original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java Tue Feb 3 07:06:24 2009
@@ -39,26 +39,26 @@
private volatile boolean stopping = false;
public String getCurrentStatus() throws AdaptorException {
- return Integer.toString(bytesPerSec);
+ return type.trim() + " " + bytesPerSec + " " + offset;
}
- public void start(long adaptor, String type, String status, long offset, ChunkReceiver dest) throws AdaptorException
+ public void start(long adaptor, String type, String bytesPerSecParam, long offset, ChunkReceiver dest) throws AdaptorException
{
try{
- bytesPerSec = Integer.parseInt(status);
+ bytesPerSec = Integer.parseInt(bytesPerSecParam.trim());
} catch(NumberFormatException e) {
- throw new AdaptorException("bad argument to const rate adaptor: " + status);
+ throw new AdaptorException("bad argument to const rate adaptor: [" + bytesPerSecParam + "]");
}
this.adaptorID = adaptor;
this.offset = offset;
this.type = type;
this.dest = dest;
- this.setName("ConstRate Adaptor");
+ this.setName("ConstRate Adaptor_" + type);
super.start(); //this is a Thread.start
}
public String getStreamName() {
- return ""+bytesPerSec;
+ return this.type;
}
public void run()
Added: hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestCharFileTailingAdaptorUTF8.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestCharFileTailingAdaptorUTF8.java?rev=740237&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestCharFileTailingAdaptorUTF8.java (added)
+++ hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestCharFileTailingAdaptorUTF8.java Tue Feb 3 07:06:24 2009
@@ -0,0 +1,63 @@
+package org.apache.hadoop.chukwa.datacollection.adaptor.filetailer;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.chukwa.Chunk;
+import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
+import org.apache.hadoop.chukwa.datacollection.connector.ChunkCatcherConnector;
+
+public class TestCharFileTailingAdaptorUTF8 extends TestCase {
+ ChunkCatcherConnector chunks;
+
+ public TestCharFileTailingAdaptorUTF8() {
+ chunks = new ChunkCatcherConnector();
+ chunks.start();
+ }
+
+ public void testCrSepAdaptor() throws IOException, InterruptedException,
+ ChukwaAgent.AlreadyRunningException {
+ ChukwaAgent agent = new ChukwaAgent();
+ File testFile = makeTestFile("/tmp/chukwaTest", 80);
+ long adaptorId = agent
+ .processCommand("add org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8"
+ + " lines " + testFile + " 0");
+ assertTrue(adaptorId != -1);
+ System.out.println("getting a chunk...");
+ Chunk c = chunks.waitForAChunk();
+ System.out.println("got chunk");
+ assertTrue(c.getSeqID() == testFile.length());
+
+ assertTrue(c.getRecordOffsets().length == 80);
+ int recStart = 0;
+ for (int rec = 0; rec < c.getRecordOffsets().length; ++rec) {
+ String record = new String(c.getData(), recStart,
+ c.getRecordOffsets()[rec] - recStart + 1);
+ System.out.println("record " + rec + " was: " + record);
+ assertTrue(record.equals(rec + " abcdefghijklmnopqrstuvwxyz\n"));
+ recStart = c.getRecordOffsets()[rec] + 1;
+ }
+ assertTrue(c.getDataType().equals("lines"));
+ agent.stopAdaptor(adaptorId, false);
+ agent.shutdown();
+ }
+
+ private File makeTestFile(String name, int size) throws IOException {
+ File tmpOutput = new File(name);
+ FileOutputStream fos = new FileOutputStream(tmpOutput);
+
+ PrintWriter pw = new PrintWriter(fos);
+ for (int i = 0; i < size; ++i) {
+ pw.print(i + " ");
+ pw.println("abcdefghijklmnopqrstuvwxyz");
+ }
+ pw.flush();
+ pw.close();
+ return tmpOutput;
+ }
+
+}