You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2009/02/26 01:46:46 UTC
svn commit: r747979 - in /hadoop/core/trunk/src/contrib/chukwa/src:
java/org/apache/hadoop/chukwa/datacollection/writer/
test/org/apache/hadoop/chukwa/datacollection/collector/
Author: asrabkin
Date: Thu Feb 26 00:46:46 2009
New Revision: 747979
URL: http://svn.apache.org/viewvc?rev=747979&view=rev
Log:
HADOOP-5033 Simplify Chukwa writer API.
Contributed by Ari Rabkin (asrabkin)
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ChukwaWriter.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/Dedup.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/PipelineStageWriter.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/collector/CaptureWriter.java
Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ChukwaWriter.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ChukwaWriter.java?rev=747979&r1=747978&r2=747979&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ChukwaWriter.java (original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ChukwaWriter.java Thu Feb 26 00:46:46 2009
@@ -26,7 +26,6 @@
public interface ChukwaWriter
{
public void init(Configuration c) throws WriterException;
- public void add(Chunk data) throws WriterException;
public void add(List<Chunk> chunks) throws WriterException;
public void close() throws WriterException;;
Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/Dedup.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/Dedup.java?rev=747979&r1=747978&r2=747979&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/Dedup.java (original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/Dedup.java Thu Feb 26 00:46:46 2009
@@ -19,6 +19,7 @@
import java.util.ArrayDeque;
import java.util.HashSet;
+import java.util.ArrayList;
import java.util.List;
import java.util.Queue;
@@ -101,16 +102,14 @@
}
@Override
- public void add(Chunk data) throws WriterException {
- if(! cache.addAndCheck(new DedupKey(data.getStreamName(), data.getSeqID())))
- next.add(data);
- }
-
- @Override
public void add(List<Chunk> chunks) throws WriterException {
+ ArrayList<Chunk> passedThrough = new ArrayList<Chunk>();
for(Chunk c: chunks)
- add(c);
-
+ if(! cache.addAndCheck(new DedupKey(c.getStreamName(), c.getSeqID())))
+ passedThrough.add(c);
+
+ if(!passedThrough.isEmpty())
+ next.add(passedThrough);
}
@Override
Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/PipelineStageWriter.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/PipelineStageWriter.java?rev=747979&r1=747978&r2=747979&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/PipelineStageWriter.java (original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/PipelineStageWriter.java Thu Feb 26 00:46:46 2009
@@ -31,12 +31,6 @@
ChukwaWriter writer; //head of pipeline
@Override
- public void add(Chunk data) throws WriterException {
- writer.add(data);
-
- }
-
- @Override
public void add(List<Chunk> chunks) throws WriterException {
writer.add(chunks);
}
Modified: hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/collector/CaptureWriter.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/collector/CaptureWriter.java?rev=747979&r1=747978&r2=747979&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/collector/CaptureWriter.java (original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/collector/CaptureWriter.java Thu Feb 26 00:46:46 2009
@@ -29,17 +29,12 @@
static ArrayList<Chunk> outputs = new ArrayList<Chunk>();
@Override
- public void add(Chunk data) throws WriterException {
- synchronized(outputs) {
- outputs.add(data);
- }
-
- }
-
- @Override
public void add(List<Chunk> chunks) throws WriterException {
+
+ synchronized(outputs) {
for(Chunk c: chunks)
- add(c);
+ outputs.add(c);
+ }
}
@Override