You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flume.apache.org by es...@apache.org on 2011/08/12 02:46:06 UTC
svn commit: r1156864 - in /incubator/flume/branches/flume-728/flume-ng-core:
./ src/ src/main/ src/main/java/ src/main/java/org/
src/main/java/org/apache/ src/main/java/org/apache/flume/
src/main/java/org/apache/flume/core/ src/main/java/org/apache/flu...
Author: esammer
Date: Fri Aug 12 00:46:05 2011
New Revision: 1156864
URL: http://svn.apache.org/viewvc?rev=1156864&view=rev
Log:
- Initial commit of reorganized core.
Added:
incubator/flume/branches/flume-728/flume-ng-core/
incubator/flume/branches/flume-728/flume-ng-core/pom.xml
incubator/flume/branches/flume-728/flume-ng-core/src/
incubator/flume/branches/flume-728/flume-ng-core/src/main/
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Context.java
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Event.java
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/EventSink.java
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/EventSource.java
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Reporter.java
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/SimpleEvent.java
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/util/
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/util/EventBuilder.java
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractEventSink.java
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/source/
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/source/AbstractEventSource.java
incubator/flume/branches/flume-728/flume-ng-core/src/test/
incubator/flume/branches/flume-728/flume-ng-core/src/test/java/
incubator/flume/branches/flume-728/flume-ng-core/src/test/java/org/
incubator/flume/branches/flume-728/flume-ng-core/src/test/java/org/apache/
incubator/flume/branches/flume-728/flume-ng-core/src/test/java/org/apache/flume/
incubator/flume/branches/flume-728/flume-ng-core/src/test/java/org/apache/flume/core/
incubator/flume/branches/flume-728/flume-ng-core/src/test/java/org/apache/flume/core/TestContext.java
incubator/flume/branches/flume-728/flume-ng-core/src/test/java/org/apache/flume/core/TestReporter.java
incubator/flume/branches/flume-728/flume-ng-core/src/test/resources/
incubator/flume/branches/flume-728/flume-ng-core/src/test/resources/log4j.properties
Added: incubator/flume/branches/flume-728/flume-ng-core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/pom.xml?rev=1156864&view=auto
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/pom.xml (added)
+++ incubator/flume/branches/flume-728/flume-ng-core/pom.xml Fri Aug 12 00:46:05 2011
@@ -0,0 +1,38 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>flume</artifactId>
+ <groupId>com.cloudera</groupId>
+ <version>0.9.5-SNAPSHOT</version>
+ </parent>
+
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-core</artifactId>
+ <version>0.9.5-SNAPSHOT</version>
+ <name>Flume NG Core</name>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+</project>
Added: incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Context.java
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Context.java?rev=1156864&view=auto
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Context.java (added)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Context.java Fri Aug 12 00:46:05 2011
@@ -0,0 +1,39 @@
+package org.apache.flume.core;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class Context {
+
+ private Map<String, Object> parameters;
+
+ public Context() {
+ parameters = new HashMap<String, Object>();
+ }
+
+ public void put(String key, Object value) {
+ parameters.put(key, value);
+ }
+
+ public <T> T get(String key, Class<? extends T> clazz) {
+ if (parameters.containsKey(key)) {
+ return clazz.cast(parameters.get(key));
+ }
+
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return "{ parameters:" + parameters + " }";
+ }
+
+ public Map<String, Object> getParameters() {
+ return parameters;
+ }
+
+ public void setParameters(Map<String, Object> parameters) {
+ this.parameters = parameters;
+ }
+
+}
Added: incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Event.java
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Event.java?rev=1156864&view=auto
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Event.java (added)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Event.java Fri Aug 12 00:46:05 2011
@@ -0,0 +1,15 @@
+package org.apache.flume.core;
+
+import java.util.Map;
+
+public interface Event<T> {
+
+ public Map<String, String> getHeaders();
+
+ public void setHeaders(Map<String, String> headers);
+
+ public T getBody();
+
+ public void setBody(T body);
+
+}
Added: incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/EventSink.java
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/EventSink.java?rev=1156864&view=auto
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/EventSink.java (added)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/EventSink.java Fri Aug 12 00:46:05 2011
@@ -0,0 +1,11 @@
+package org.apache.flume.core;
+
+public interface EventSink {
+
+ public void open(Context context);
+
+ public void append(Context context, Event<?> event);
+
+ public void close(Context context);
+
+}
Added: incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/EventSource.java
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/EventSource.java?rev=1156864&view=auto
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/EventSource.java (added)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/EventSource.java Fri Aug 12 00:46:05 2011
@@ -0,0 +1,11 @@
+package org.apache.flume.core;
+
+public interface EventSource {
+
+ public void open(Context context);
+
+ public Event<?> next(Context context);
+
+ public void close(Context context);
+
+}
Added: incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Reporter.java
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Reporter.java?rev=1156864&view=auto
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Reporter.java (added)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/Reporter.java Fri Aug 12 00:46:05 2011
@@ -0,0 +1,35 @@
+package org.apache.flume.core;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Reporter {
+
+ private static final Logger logger = LoggerFactory.getLogger(Reporter.class);
+
+ private long timeStamp;
+
+ public Reporter() {
+ timeStamp = 0;
+ }
+
+ public synchronized long progress() {
+ long previousTimeStamp = timeStamp;
+ timeStamp = System.currentTimeMillis();
+
+ logger.debug("progress updated. previousTimeStamp:{} timeStamp:{}",
+ previousTimeStamp, timeStamp);
+
+ return timeStamp;
+ }
+
+ public synchronized long getTimeStamp() {
+ return timeStamp;
+ }
+
+ @Override
+ public String toString() {
+ return "{ timeStamp:" + timeStamp + " }";
+ }
+
+}
Added: incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/SimpleEvent.java
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/SimpleEvent.java?rev=1156864&view=auto
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/SimpleEvent.java (added)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/SimpleEvent.java Fri Aug 12 00:46:05 2011
@@ -0,0 +1,41 @@
+package org.apache.flume.core;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class SimpleEvent<T> implements Event<T> {
+
+ private Map<String, String> headers;
+ private T body;
+
+ public SimpleEvent() {
+ headers = new HashMap<String, String>();
+ body = null;
+ }
+
+ @Override
+ public Map<String, String> getHeaders() {
+ return headers;
+ }
+
+ @Override
+ public void setHeaders(Map<String, String> headers) {
+ this.headers = headers;
+ }
+
+ @Override
+ public T getBody() {
+ return body;
+ }
+
+ @Override
+ public void setBody(T body) {
+ this.body = body;
+ }
+
+ @Override
+ public String toString() {
+ return "{ headers:" + headers + " body:" + body + " }";
+ }
+
+}
Added: incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/util/EventBuilder.java
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/util/EventBuilder.java?rev=1156864&view=auto
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/util/EventBuilder.java (added)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/core/util/EventBuilder.java Fri Aug 12 00:46:05 2011
@@ -0,0 +1,26 @@
+package org.apache.flume.core.util;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.flume.core.Event;
+import org.apache.flume.core.SimpleEvent;
+
+public class EventBuilder {
+
+ public static <T> Event<T> withBody(T body) {
+ Event<T> event = new SimpleEvent<T>();
+
+ return event;
+ }
+
+ public static <T> Event<T> withBody(T body, Map<String, String> headers) {
+ Event<T> event = new SimpleEvent<T>();
+
+ event.setBody(body);
+ event.setHeaders(new HashMap<String, String>(headers));
+
+ return event;
+ }
+
+}
Added: incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractEventSink.java
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractEventSink.java?rev=1156864&view=auto
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractEventSink.java (added)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractEventSink.java Fri Aug 12 00:46:05 2011
@@ -0,0 +1,22 @@
+package org.apache.flume.sink;
+
+import org.apache.flume.core.Context;
+import org.apache.flume.core.Event;
+import org.apache.flume.core.EventSink;
+
+abstract public class AbstractEventSink implements EventSink {
+
+ @Override
+ public void open(Context context) {
+ // Empty implementation by default.
+ }
+
+ @Override
+ abstract public void append(Context context, Event<?> event);
+
+ @Override
+ public void close(Context context) {
+ // Empty implementation by default.
+ }
+
+}
Added: incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/source/AbstractEventSource.java
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/source/AbstractEventSource.java?rev=1156864&view=auto
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/source/AbstractEventSource.java (added)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/source/AbstractEventSource.java Fri Aug 12 00:46:05 2011
@@ -0,0 +1,22 @@
+package org.apache.flume.source;
+
+import org.apache.flume.core.Context;
+import org.apache.flume.core.Event;
+import org.apache.flume.core.EventSource;
+
+abstract public class AbstractEventSource implements EventSource {
+
+ @Override
+ public void open(Context context) {
+ // Empty implementation by default.
+ }
+
+ @Override
+ abstract public Event<?> next(Context context);
+
+ @Override
+ public void close(Context context) {
+ // Empty implementation by default.
+ }
+
+}
Added: incubator/flume/branches/flume-728/flume-ng-core/src/test/java/org/apache/flume/core/TestContext.java
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/test/java/org/apache/flume/core/TestContext.java?rev=1156864&view=auto
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/test/java/org/apache/flume/core/TestContext.java (added)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/test/java/org/apache/flume/core/TestContext.java Fri Aug 12 00:46:05 2011
@@ -0,0 +1,27 @@
+package org.apache.flume.core;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestContext {
+
+ private Context context;
+
+ @Before
+ public void setUp() {
+ context = new Context();
+ }
+
+ @Test
+ public void testPutGet() {
+ Assert.assertEquals("Context is empty", 0, context.getParameters()
+ .size());
+
+ context.put("test", "test");
+
+ Assert.assertEquals("Context contains test value", "test",
+ context.get("test", String.class));
+ }
+
+}
Added: incubator/flume/branches/flume-728/flume-ng-core/src/test/java/org/apache/flume/core/TestReporter.java
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/test/java/org/apache/flume/core/TestReporter.java?rev=1156864&view=auto
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/test/java/org/apache/flume/core/TestReporter.java (added)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/test/java/org/apache/flume/core/TestReporter.java Fri Aug 12 00:46:05 2011
@@ -0,0 +1,22 @@
+package org.apache.flume.core;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestReporter {
+
+ private Reporter reporter;
+
+ @Before
+ public void setUp() {
+ reporter = new Reporter();
+ }
+
+ @Test
+ public void testProgress() {
+ Assert.assertEquals("Timestamp starts at zero", 0, reporter.getTimeStamp());
+ Assert.assertTrue("Timestamp > 0 after progress", reporter.progress() > 0);
+ }
+
+}
Added: incubator/flume/branches/flume-728/flume-ng-core/src/test/resources/log4j.properties
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/test/resources/log4j.properties?rev=1156864&view=auto
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/test/resources/log4j.properties (added)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/test/resources/log4j.properties Fri Aug 12 00:46:05 2011
@@ -0,0 +1,7 @@
+log4j.rootLogger = INFO, out
+
+log4j.appender.out = org.apache.log4j.ConsoleAppender
+log4j.appender.out.layout = org.apache.log4j.PatternLayout
+log4j.appender.out.layout.ConversionPattern = %d (%t) [%l - %p] %m%n
+
+log4j.logger.org.apache.flume = DEBUG