You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@chukwa.apache.org by as...@apache.org on 2009/08/20 22:55:48 UTC
svn commit: r806355 - in /hadoop/chukwa/trunk/src:
java/org/apache/hadoop/chukwa/extraction/archive/
test/org/apache/hadoop/chukwa/datacollection/collector/
test/org/apache/hadoop/chukwa/datacollection/controller/
Author: asrabkin
Date: Thu Aug 20 20:55:48 2009
New Revision: 806355
URL: http://svn.apache.org/viewvc?rev=806355&view=rev
Log:
CHUKWA-373. Fix Backpressure test case
Modified:
hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java
hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java
hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java
Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java?rev=806355&r1=806354&r2=806355&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java Thu Aug 20 20:55:48 2009
@@ -91,6 +91,9 @@
fs.mkdirs(pMRInputDir);
Path pOutputDir = new Path(archivesMROutputDir);
+ if(!fs.exists(pOutputDir))
+ fs.mkdirs(pOutputDir);
+
if(fs.listStatus(pOutputDir).length == 0)
fs.delete(pOutputDir, true);
Path archive = new Path(chukwaRootDir + "archive");
Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java?rev=806355&r1=806354&r2=806355&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java (original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java Thu Aug 20 20:55:48 2009
@@ -15,15 +15,24 @@
public class TestBackpressure extends TestCase {
int PORTNO = 9991;
- int SLEEP_SEC = 20;
- int WRITE_RATE = 300; //kb/sec
- int SEND_RATE = 2500000; //bytes/sec
+
+ /**
+ * NOTE THAT WRITE-RATE * POST SIZE MUST BE GREATER THAN TEST DURATION
+ *
+ * Default max post size is 2 MB; need to process that several times during test.
+ */
+ int TEST_DURATION_SECS = 40;
+ int WRITE_RATE_KB = 200; //kb/sec
+
+
+ int SEND_RATE = 2500* 1000; //bytes/sec
+ int MIN_ACCEPTABLE_PERCENT = 60;
public void testBackpressure() throws Exception {
Configuration conf = new Configuration();
conf.set("chukwaCollector.writerClass", NullWriter.class
.getCanonicalName());
- conf.set(NullWriter.RATE_OPT_NAME, ""+WRITE_RATE);//kb/sec
+ conf.set(NullWriter.RATE_OPT_NAME, ""+WRITE_RATE_KB);//kb/sec
conf.setInt(HttpConnector.MIN_POST_INTERVAL_OPT, 100);
conf.setInt("constAdaptor.sleepVariance", 1);
conf.setInt("constAdaptor.minSleep", 50);
@@ -41,13 +50,13 @@
Thread.sleep(1000);
agent.processAddCommand("add constSend = " + ConstRateAdaptor.class.getCanonicalName() +
" testData "+ SEND_RATE + " 0");
- Thread.sleep(SLEEP_SEC * 1000);
+ Thread.sleep(TEST_DURATION_SECS * 1000);
String stat = agent.getAdaptorList().get("constSend");
- long bytesPerSec = Long.valueOf(stat.split(" ")[3]) / SLEEP_SEC / 1000;
- System.out.println("data rate was " + bytesPerSec + " kb /second");
- assertTrue(bytesPerSec < WRITE_RATE);
- assertTrue(bytesPerSec > 3* WRITE_RATE / 4);//an assumption, but should hold true
+ long kbytesPerSec = Long.valueOf(stat.split(" ")[3]) / TEST_DURATION_SECS / 1000;
+ System.out.println("data rate was " + kbytesPerSec + " kb /second");
+ assertTrue(kbytesPerSec < WRITE_RATE_KB); //write rate should throttle sends
+ assertTrue(kbytesPerSec > MIN_ACCEPTABLE_PERCENT* WRITE_RATE_KB / 100);//an assumption, but should hold true
agent.shutdown();
}
}
Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java?rev=806355&r1=806354&r2=806355&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java (original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java Thu Aug 20 20:55:48 2009
@@ -18,8 +18,9 @@
package org.apache.hadoop.chukwa.datacollection.controller;
-import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
+import org.apache.hadoop.chukwa.datacollection.connector.ChunkCatcherConnector;
import org.apache.hadoop.chukwa.datacollection.connector.Connector;
import org.apache.hadoop.chukwa.datacollection.connector.http.HttpConnector;
import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
@@ -28,30 +29,24 @@
import junit.framework.TestCase;
public class TestAgentClient extends TestCase {
- ChukwaConfiguration config;
+ Configuration config;
ChukwaAgent agent;
ChukwaAgentController c;
- Connector httpConnector;
+ Connector connector;
// consoleConnector = new ConsoleOutConnector(agent);
protected void setUp() throws ChukwaAgent.AlreadyRunningException {
- config = new ChukwaConfiguration();
- agent = new ChukwaAgent();
+ config = new Configuration();
+ agent = new ChukwaAgent(config);
c = new ChukwaAgentController();
- httpConnector = new HttpConnector(agent); // use default source for list of
- // collectors (i.e.
- // conf/connectors)
-
- httpConnector.start();
-
- // assertTrue(Integer.parseInt(config.get("chukwaAgent.control.port")) ==
- // agent.getControlSock().getPortNumber());
+ connector = new ChunkCatcherConnector();
+ connector.start();
}
protected void tearDown() {
System.out.println("in tearDown()");
- ((HttpConnector) httpConnector).shutdown();
+ connector.shutdown();
}
public void testAddFile() {