You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@logging.apache.org by rg...@apache.org on 2026/05/25 21:42:04 UTC
(logging-flume) branch extract-rpc created (now e3a4e3f9b)
This is an automated email from the ASF dual-hosted git repository.
rgoers pushed a change to branch extract-rpc
in repository https://gitbox.apache.org/repos/asf/logging-flume.git
at e3a4e3f9b Move Avro & Thrift. Switch to java.time from joda
This branch includes the following new commits:
new e3a4e3f9b Move Avro & Thrift. Switch to java.time from joda
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(logging-flume) 01/01: Move Avro & Thrift. Switch to java.time from joda
Posted by rg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rgoers pushed a commit to branch extract-rpc
in repository https://gitbox.apache.org/repos/asf/logging-flume.git
commit e3a4e3f9b62aa8cde7505a876939d0274c61b170
Author: rgoers <rg...@apache.org>
AuthorDate: Mon May 25 14:41:33 2026 -0700
Move Avro & Thrift. Switch to java.time from joda
---
bin/flume-ng | 2 +-
build-support/pom.xml | 2 +-
flume-bom/pom.xml | 47 +-
flume-ng-auth/pom.xml | 2 +-
flume-ng-channels/flume-file-channel/pom.xml | 2 +-
.../org/apache/flume/channel/file/FileChannel.java | 4 +-
.../apache/flume/channel/file/TestFileChannel.java | 2 +-
.../channel/file/TestFileChannelErrorMetrics.java | 2 +-
.../org/apache/flume/channel/file/TestUtils.java | 2 +-
.../file/encryption/TestFileChannelEncryption.java | 2 +-
.../flume-spillable-memory-channel/pom.xml | 2 +-
.../flume/channel/SpillableMemoryChannel.java | 4 +-
.../flume/channel/TestSpillableMemoryChannel.java | 4 +-
flume-ng-channels/pom.xml | 2 +-
flume-ng-clients/flume-ng-log4jappender/pom.xml | 2 +-
.../TestLoadBalancingLog4jAppender.java | 1 -
.../clients/log4jappender/TestLog4jAppender.java | 1 -
.../log4jappender/TestLog4jAppenderWithAvro.java | 1 -
flume-ng-clients/pom.xml | 2 +-
.../flume-ng-config-filter-api/pom.xml | 2 +-
.../pom.xml | 2 +-
.../pom.xml | 2 +-
flume-ng-configfilters/pom.xml | 2 +-
flume-ng-configuration/pom.xml | 2 +-
.../java/org/apache/flume/conf/sink/SinkType.java | 21 +-
.../org/apache/flume/conf/source/SourceType.java | 6 +-
flume-ng-core/pom.xml | 115 +---
.../src/main/java/org/apache/flume/Channel.java | 1 +
.../flume/channel/BasicChannelSemantics.java | 2 +-
.../flume/channel/BasicTransactionSemantics.java | 2 +-
.../org/apache/flume/channel/ChannelProcessor.java | 2 +-
.../org/apache/flume/channel/ChannelUtils.java | 2 +-
.../org/apache/flume/channel/MemoryChannel.java | 4 +-
.../flume/channel/PseudoTxnMemoryChannel.java | 2 +-
.../formatter/output/RollTimePathManager.java | 9 +-
.../RegexExtractorInterceptorMillisSerializer.java | 12 +-
.../flume/serialization/EventDeserializerType.java | 20 +-
.../flume/serialization/EventSerializerType.java | 20 +-
.../serialization/TransientPositionTracker.java | 0
.../org/apache/flume/sink/AbstractRpcSink.java | 4 +-
.../java/org/apache/flume/source/NetcatSource.java | 2 +-
.../org/apache/flume/source/NetcatUdpSource.java | 2 +-
.../flume/source/SequenceGeneratorSource.java | 2 +-
.../java/org/apache/flume/source/StressSource.java | 2 +-
.../java/org/apache/flume/source/SyslogParser.java | 51 +-
.../org/apache/flume/source/SyslogTcpSource.java | 2 +-
.../org/apache/flume/source/SyslogUDPSource.java | 2 +-
.../org/apache/flume/source/http/HTTPSource.java | 2 +-
.../channel/AbstractBasicChannelSemanticsTest.java | 2 +-
.../java/org/apache/flume/channel/MockChannel.java | 2 +-
.../apache/flume/channel/TestChannelProcessor.java | 2 +-
.../org/apache/flume/channel/TestChannelUtils.java | 2 +-
.../apache/flume/channel/TestMemoryChannel.java | 2 +-
.../channel/TestMemoryChannelConcurrency.java | 2 +-
.../flume/formatter/output/TestBucketPath.java | 12 +-
.../interceptor/TestRegexExtractorInterceptor.java | 15 +-
...tRegexExtractorInterceptorMillisSerializer.java | 14 +-
.../apache/flume/sink/TestDefaultSinkFactory.java | 24 +-
.../flume/sink/TestLoadBalancingSinkProcessor.java | 2 +-
.../org/apache/flume/sink/TestRollingFileSink.java | 2 +-
.../org/apache/flume/source/TestAvroSource.java | 610 --------------------
.../flume/source/TestDefaultSourceFactory.java | 29 +-
.../flume/source/TestMultiportSyslogTCPSource.java | 13 +-
.../org/apache/flume/source/TestNetcatSource.java | 2 +-
.../flume/source/TestSequenceGeneratorSource.java | 2 +-
.../org/apache/flume/source/TestStressSource.java | 2 +-
.../org/apache/flume/source/TestSyslogParser.java | 49 +-
.../apache/flume/source/TestSyslogTcpSource.java | 6 +-
.../apache/flume/source/TestSyslogUdpSource.java | 7 +-
.../org/apache/flume/source/TestThriftSource.java | 383 -------------
flume-ng-dist/pom.xml | 32 +-
flume-ng-embedded-agent/pom.xml | 8 +-
.../apache/flume/agent/embedded/EmbeddedAgent.java | 2 +-
.../flume/agent/embedded/EmbeddedSource.java | 2 +-
flume-ng-node/pom.xml | 6 +-
.../node/TestAbstractConfigurationProvider.java | 52 +-
flume-ng-sdk/pom.xml | 175 +-----
.../src/main/java/org/apache/flume/Context.java | 0
.../org/apache/flume/annotations/Disposable.java | 0
.../flume/annotations/InterfaceAudience.java | 0
.../flume/annotations/InterfaceStability.java | 0
.../org/apache/flume/annotations/Recyclable.java | 0
.../org/apache/flume/api/FailoverRpcClient.java | 2 +-
.../org/apache/flume/api/RpcClientFactory.java | 78 ++-
.../java/org/apache/flume/event}/EventReader.java | 2 +-
.../apache/flume/event}/ReliableEventReader.java | 2 +-
.../flume/event}/SimpleTextLineEventReader.java | 3 +-
.../apache/flume/exception}/ChannelException.java | 2 +-
.../flume/exception}/ChannelFullException.java | 2 +-
.../flume/instrumentation/ChannelCounter.java | 0
.../flume/instrumentation/ChannelCounterMBean.java | 0
.../instrumentation/MonitoredCounterGroup.java | 0
.../apache/flume/instrumentation/SinkCounter.java | 2 +-
.../flume/instrumentation/SinkCounterMBean.java | 0
.../flume/instrumentation/SourceCounter.java | 2 +-
.../flume/instrumentation/SourceCounterMBean.java | 0
.../flume/serialization/DecodeErrorPolicy.java | 0
.../flume/serialization/EventDeserializer.java | 0
.../flume/serialization/LengthMeasurable.java | 0
.../flume/serialization/LineDeserializer.java | 0
.../flume/serialization/PositionTracker.java | 0
.../apache/flume/serialization/RemoteMarkable.java | 0
.../org/apache/flume/serialization/Resettable.java | 0
.../serialization/ResettableFileInputStream.java | 0
.../flume/serialization/ResettableInputStream.java | 0
.../org/apache/flume/serialization/Seekable.java | 0
...SpoolDirectorySourceConfigurationConstants.java | 0
.../org/apache/flume/tools/PlatformDetect.java | 0
.../org/apache/flume/api/ThriftTestingSource.java | 282 ---------
.../flume/event}/TestBufferedLineReader.java | 18 +-
flume-ng-sinks/flume-http-sink/pom.xml | 2 +-
flume-ng-sinks/flume-irc-sink/pom.xml | 2 +-
.../java/org/apache/flume/sink/irc/IRCSink.java | 2 +-
flume-ng-sinks/pom.xml | 2 +-
flume-ng-sources/flume-jms-source/pom.xml | 2 +-
.../org/apache/flume/source/jms/JMSSource.java | 2 +-
.../org/apache/flume/source/jms/TestJMSSource.java | 2 +-
flume-ng-sources/flume-scribe-source/pom.xml | 2 +-
flume-ng-sources/flume-taildir-source/pom.xml | 2 +-
.../source/taildir/ReliableTaildirEventReader.java | 2 +-
.../apache/flume/source/taildir/TaildirSource.java | 2 +-
.../flume/source/taildir/TestTaildirSource.java | 2 +-
flume-ng-sources/pom.xml | 3 +-
flume-ng-tests/pom.xml | 6 +-
flume-ng-tests/src/test/resources/log4j2.xml | 2 +-
flume-parent/pom.xml | 154 +----
flume-rpc/flume-rpc-avro/pom.xml | 222 +++++++
.../src/main/avro/TransferStateFileMeta.avsc | 0
.../flume-rpc-avro}/src/main/avro/flume.avdl | 0
.../flume/rpc/avro/client}/AvroCLIClient.java | 6 +-
.../flume/rpc/avro/client}/NettyAvroRpcClient.java | 15 +-
.../serialization/AbstractAvroEventSerializer.java | 8 +-
.../avro}/serialization/AvroEventDeserializer.java | 6 +-
.../AvroEventSerializerConfigurationConstants.java | 2 +-
.../serialization/DurablePositionTracker.java | 4 +-
.../FlumeEventAvroEventSerializer.java | 3 +-
.../ReliableSpoolingFileEventReader.java | 4 +-
.../org/apache/flume/rpc/avro}/sink/AvroSink.java | 9 +-
.../apache/flume/rpc/avro}/source/AvroSource.java | 25 +-
.../rpc/avro}/source/SpoolDirectorySource.java | 9 +-
.../flume/rpc}/api/TestFailoverRpcClient.java | 7 +-
.../org/apache/flume/rpc/avro}/RpcTestUtils.java | 6 +-
.../rpc/avro}/TestLoadBalancingRpcClient.java | 9 +-
.../flume/rpc/avro}/TestRpcClientFactory.java | 6 +-
.../rpc/avro/client}/TestNettyAvroRpcClient.java | 12 +-
.../serialization/SyslogAvroEventSerializer.java | 32 +-
.../serialization/TestAvroEventDeserializer.java | 6 +-
.../serialization/TestDurablePositionTracker.java | 3 +-
.../TestFlumeEventAvroEventSerializer.java | 3 +-
.../TestReliableSpoolingFileEventReader.java | 33 +-
.../TestResettableFileInputStream.java | 6 +-
.../serialization}/TestSpoolingFileLineReader.java | 2 +-
.../TestSyslogAvroEventSerializer.java | 4 +-
.../apache/flume/rpc/avro}/sink/TestAvroSink.java | 6 +-
.../rpc/avro}/sink/TestDefaultSinkFactory.java | 6 +-
.../apache/flume/rpc/avro/source/MockSource.java | 51 +-
.../rpc/avro}/source/TestDefaultSourceFactory.java | 11 +-
.../rpc/avro}/source/TestSpoolDirectorySource.java | 9 +-
.../node/TestAbstractConfigurationProvider.java | 6 +-
.../resources/TestResettableFileInputStream_1.avro | Bin
.../TestResettableFileInputStream_1.truncated.avro | Bin
.../src/test/resources/certs/gencerts.sh | 31 +
.../src/test/resources/certs/rootca.conf | 9 +
.../src/test/resources/certs/server.conf | 9 +
.../src/test/resources/jettykeystore | Bin 0 -> 1355 bytes
.../src/test/resources/keystorefile.jks | Bin 0 -> 3518 bytes
.../flume-rpc-avro}/src/test/resources/log4j2.xml | 2 +-
.../src/test/resources/server.flume-crt.pem | 17 +
.../src/test/resources/server.flume-keystore.p12 | Bin 0 -> 2447 bytes
.../src/test/resources/server.flume.pem | 23 +
.../flume-rpc-avro/src/test/resources/server.p12 | Bin 0 -> 1637 bytes
.../src/test/resources/syslog_event.avsc | 0
.../src/test/resources/truststore.jks | Bin 0 -> 687 bytes
.../src/test/resources/truststorefile.jks | Bin 0 -> 775 bytes
flume-rpc/flume-rpc-thrift/pom.xml | 216 +++++++
.../java/org/apache/flume/rpc}/thrift/Status.java | 4 +-
.../apache/flume/rpc}/thrift/ThriftFlumeEvent.java | 5 +-
.../flume/rpc}/thrift/ThriftSourceProtocol.java | 16 +-
.../rpc/thrift}/api/SecureRpcClientFactory.java | 5 +-
.../rpc/thrift}/api/SecureThriftRpcClient.java | 5 +-
.../flume/rpc/thrift/client}/ThriftRpcClient.java | 20 +-
.../apache/flume/rpc/thrift}/sink/ThriftSink.java | 9 +-
.../flume/rpc/thrift}/source/ThriftSource.java | 21 +-
.../flume-rpc-thrift}/src/main/thrift/aslv2 | 0
.../flume-rpc-thrift}/src/main/thrift/flume.thrift | 0
.../flume/rpc/thrift}/TestThriftRpcClient.java | 6 +-
.../flume/rpc/thrift/ThriftTestingSource.java | 636 +++++++++++++++++++++
.../flume/rpc/thrift}/sink/TestThriftSink.java | 6 +-
.../apache/flume/rpc/thrift/source/MockSource.java | 51 +-
.../thrift}/source/TestDefaultSourceFactory.java | 35 +-
.../src/test/resources/certs/gencerts.sh | 31 +
.../src/test/resources/certs/rootca.conf | 9 +
.../src/test/resources/certs/server.conf | 9 +
.../src/test/resources/keystorefile.jks | Bin 0 -> 3518 bytes
.../src/test/resources/log4j2.xml | 2 +-
.../src/test/resources/truststore.jks | Bin 0 -> 687 bytes
.../src/test/resources/truststorefile.jks | Bin 0 -> 775 bytes
flume-rpc/pom.xml | 162 ++++++
flume-tools/pom.xml | 2 +-
pom.xml | 4 +-
200 files changed, 2083 insertions(+), 2221 deletions(-)
diff --git a/bin/flume-ng b/bin/flume-ng
index fc28ba272..b575dcda5 100755
--- a/bin/flume-ng
+++ b/bin/flume-ng
@@ -24,7 +24,7 @@
################################
FLUME_AGENT_CLASS="org.apache.flume.node.Application"
-FLUME_AVRO_CLIENT_CLASS="org.apache.flume.client.avro.AvroCLIClient"
+FLUME_AVRO_CLIENT_CLASS="org.apache.flume.rpc.avro.client.AvroCLIClient"
FLUME_VERSION_CLASS="org.apache.flume.tools.VersionInfo"
FLUME_TOOLS_CLASS="org.apache.flume.tools.FlumeToolsMain"
diff --git a/build-support/pom.xml b/build-support/pom.xml
index e520e82df..b14795219 100644
--- a/build-support/pom.xml
+++ b/build-support/pom.xml
@@ -22,7 +22,7 @@ limitations under the License.
<groupId>org.apache.flume</groupId>
<artifactId>build-support</artifactId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<name>Build Support</name>
<description>Build tools and Configuration</description>
<properties>
diff --git a/flume-bom/pom.xml b/flume-bom/pom.xml
index d05d2aff0..58b852b6e 100644
--- a/flume-bom/pom.xml
+++ b/flume-bom/pom.xml
@@ -21,7 +21,7 @@ limitations under the License.
<parent>
<groupId>org.apache.flume</groupId>
<artifactId>flume-project</artifactId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -38,14 +38,10 @@ limitations under the License.
<flume-jdbc.version>2.0.0-SNAPSHOT</flume-jdbc.version>
<flume-jms.version>1.11.1-SNAPSHOT</flume-jms.version>
<flume-kafka.version>2.0.0-SNAPSHOT</flume-kafka.version>
- <flume-log4jappender.version>1.11.1-SNAPSHOT</flume-log4jappender.version>
<flume-scribe.version>1.11.1-SNAPSHOT</flume-scribe.version>
<flume-spring-boot.version>2.0.0-SNAPSHOT</flume-spring-boot.version>
<flume-taildir.version>1.11.1-SNAPSHOT</flume-taildir.version>
<flume-tools.version>1.11.1-SNAPSHOT</flume-tools.version>
- <flume-twitter.version>2.0.0-SNAPSHOT</flume-twitter.version>
- <flume-legacy.version>2.0.0-SNAPSHOT</flume-legacy.version>
- <flume-morphline.version>2.0.0-SNAPSHOT</flume-morphline.version>
</properties>
<dependencyManagement>
@@ -107,7 +103,12 @@ limitations under the License.
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-sdk</artifactId>
+ <artifactId>flume-rpc-avro</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-rpc-thrift</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -145,33 +146,7 @@ limitations under the License.
<artifactId>flume-spillable-memory-channel</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.flume.flume-ng-clients</groupId>
- <artifactId>flume-ng-log4jappender</artifactId>
- <version>${flume-log4jappender.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flume.flume-ng-clients</groupId>
- <artifactId>flume-ng-log4jappender</artifactId>
- <version>${flume-log4jappender.version}</version>
- <classifier>jar-with-dependencies</classifier>
- </dependency>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-legacy-thrift-source</artifactId>
- <version>${flume-legacy.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-legacy-avro-source</artifactId>
- <version>${flume-legacy.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-morphline-solr-sink</artifactId>
- <version>${flume-morphline.version}</version>
- </dependency>
- <dependency>
+ <!-- <dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-hdfs-sink</artifactId>
<version>${flume-hadoop.version}</version>
@@ -195,17 +170,17 @@ limitations under the License.
<groupId>org.apache.flume</groupId>
<artifactId>flume-hbase2-sink</artifactId>
<version>${flume-hadoop.version}</version>
- </dependency>
+ </dependency>-->
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-kafka-sink</artifactId>
<version>${flume-kafka.version}</version>
</dependency>
- <dependency>
+ <!-- <dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-kudu-sink</artifactId>
<version>${flume-hadoop.version}</version>
- </dependency>
+ </dependency>-->
<dependency>
<groupId>org.apache.flume.flume-ng-sources</groupId>
<artifactId>flume-jms-source</artifactId>
diff --git a/flume-ng-auth/pom.xml b/flume-ng-auth/pom.xml
index 766eda5af..e213d7844 100644
--- a/flume-ng-auth/pom.xml
+++ b/flume-ng-auth/pom.xml
@@ -21,7 +21,7 @@ limitations under the License.
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../flume-parent/pom.xml</relativePath>
</parent>
diff --git a/flume-ng-channels/flume-file-channel/pom.xml b/flume-ng-channels/flume-file-channel/pom.xml
index 6bfc85f5e..6db659c86 100644
--- a/flume-ng-channels/flume-file-channel/pom.xml
+++ b/flume-ng-channels/flume-file-channel/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>flume-ng-channels</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.flume.flume-ng-channels</groupId>
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java
index fe7e80cc8..a0a47b379 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java
@@ -26,8 +26,8 @@ import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import com.google.common.collect.Iterables;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
-import org.apache.flume.ChannelFullException;
+import org.apache.flume.exception.ChannelException;
+import org.apache.flume.exception.ChannelFullException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.annotations.Disposable;
diff --git a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java
index a3d27f7de..bf5dca7bd 100644
--- a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java
+++ b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java
@@ -22,7 +22,7 @@ import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Event;
import org.apache.flume.Transaction;
import org.apache.flume.channel.file.FileChannel.FileBackedTransaction;
diff --git a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannelErrorMetrics.java b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannelErrorMetrics.java
index e2d1ee627..095fdf0f9 100644
--- a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannelErrorMetrics.java
+++ b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannelErrorMetrics.java
@@ -20,7 +20,7 @@ package org.apache.flume.channel.file;
import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Transaction;
import org.apache.flume.channel.file.instrumentation.FileChannelCounter;
import org.apache.flume.event.EventBuilder;
diff --git a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
index 057518d4a..cd40589ce 100644
--- a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
+++ b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
@@ -26,7 +26,7 @@ import com.google.common.io.ByteStreams;
import com.google.common.io.Files;
import com.google.common.io.Resources;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.Transaction;
diff --git a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/encryption/TestFileChannelEncryption.java b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/encryption/TestFileChannelEncryption.java
index 4e5ab6f88..441133d29 100644
--- a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/encryption/TestFileChannelEncryption.java
+++ b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/encryption/TestFileChannelEncryption.java
@@ -22,7 +22,7 @@ import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
import com.google.common.collect.Maps;
import com.google.common.io.Files;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.FlumeException;
import org.apache.flume.channel.file.FileChannelConfiguration;
import org.apache.flume.channel.file.TestFileChannelBase;
diff --git a/flume-ng-channels/flume-spillable-memory-channel/pom.xml b/flume-ng-channels/flume-spillable-memory-channel/pom.xml
index d89fba71b..bd7c6be17 100644
--- a/flume-ng-channels/flume-spillable-memory-channel/pom.xml
+++ b/flume-ng-channels/flume-spillable-memory-channel/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>flume-ng-channels</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.flume.flume-ng-channels</groupId>
diff --git a/flume-ng-channels/flume-spillable-memory-channel/src/main/java/org/apache/flume/channel/SpillableMemoryChannel.java b/flume-ng-channels/flume-spillable-memory-channel/src/main/java/org/apache/flume/channel/SpillableMemoryChannel.java
index 19ca710f3..9e8c499e3 100644
--- a/flume-ng-channels/flume-spillable-memory-channel/src/main/java/org/apache/flume/channel/SpillableMemoryChannel.java
+++ b/flume-ng-channels/flume-spillable-memory-channel/src/main/java/org/apache/flume/channel/SpillableMemoryChannel.java
@@ -20,8 +20,8 @@ package org.apache.flume.channel;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.flume.ChannelException;
-import org.apache.flume.ChannelFullException;
+import org.apache.flume.exception.ChannelException;
+import org.apache.flume.exception.ChannelFullException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.Transaction;
diff --git a/flume-ng-channels/flume-spillable-memory-channel/src/test/java/org/apache/flume/channel/TestSpillableMemoryChannel.java b/flume-ng-channels/flume-spillable-memory-channel/src/test/java/org/apache/flume/channel/TestSpillableMemoryChannel.java
index b0939393d..e19a28e44 100644
--- a/flume-ng-channels/flume-spillable-memory-channel/src/test/java/org/apache/flume/channel/TestSpillableMemoryChannel.java
+++ b/flume-ng-channels/flume-spillable-memory-channel/src/test/java/org/apache/flume/channel/TestSpillableMemoryChannel.java
@@ -30,8 +30,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
-import org.apache.flume.ChannelException;
-import org.apache.flume.ChannelFullException;
+import org.apache.flume.exception.ChannelException;
+import org.apache.flume.exception.ChannelFullException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.Transaction;
diff --git a/flume-ng-channels/pom.xml b/flume-ng-channels/pom.xml
index d47523673..9bf2ba17b 100644
--- a/flume-ng-channels/pom.xml
+++ b/flume-ng-channels/pom.xml
@@ -23,7 +23,7 @@ limitations under the License.
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../flume-parent/pom.xml</relativePath>
</parent>
diff --git a/flume-ng-clients/flume-ng-log4jappender/pom.xml b/flume-ng-clients/flume-ng-log4jappender/pom.xml
index 83fd9586d..6e751849a 100644
--- a/flume-ng-clients/flume-ng-log4jappender/pom.xml
+++ b/flume-ng-clients/flume-ng-log4jappender/pom.xml
@@ -21,7 +21,7 @@ limitations under the License.
<parent>
<artifactId>flume-ng-clients</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
diff --git a/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLoadBalancingLog4jAppender.java b/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLoadBalancingLog4jAppender.java
index 0546c23c1..cd8b42327 100644
--- a/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLoadBalancingLog4jAppender.java
+++ b/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLoadBalancingLog4jAppender.java
@@ -45,7 +45,6 @@ import org.apache.flume.channel.ChannelProcessor;
import org.apache.flume.channel.MemoryChannel;
import org.apache.flume.channel.ReplicatingChannelSelector;
import org.apache.flume.conf.Configurables;
-import org.apache.flume.source.AvroSource;
import org.apache.flume.source.avro.AvroFlumeEvent;
import org.apache.flume.source.avro.Status;
import org.apache.log4j.Level;
diff --git a/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppender.java b/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppender.java
index 891cb669d..960c73a2f 100644
--- a/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppender.java
+++ b/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppender.java
@@ -43,7 +43,6 @@ import org.apache.flume.channel.ChannelProcessor;
import org.apache.flume.channel.MemoryChannel;
import org.apache.flume.channel.ReplicatingChannelSelector;
import org.apache.flume.conf.Configurables;
-import org.apache.flume.source.AvroSource;
import org.apache.flume.source.avro.AvroFlumeEvent;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
diff --git a/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java b/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java
index 23c2d7e8d..25123aca6 100644
--- a/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java
+++ b/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java
@@ -48,7 +48,6 @@ import org.apache.flume.channel.ChannelProcessor;
import org.apache.flume.channel.MemoryChannel;
import org.apache.flume.channel.ReplicatingChannelSelector;
import org.apache.flume.conf.Configurables;
-import org.apache.flume.source.AvroSource;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
diff --git a/flume-ng-clients/pom.xml b/flume-ng-clients/pom.xml
index 70b5348e3..d54c46c5d 100644
--- a/flume-ng-clients/pom.xml
+++ b/flume-ng-clients/pom.xml
@@ -20,7 +20,7 @@ limitations under the License.
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../flume-parent/pom.xml</relativePath>
</parent>
<artifactId>flume-ng-clients</artifactId>
diff --git a/flume-ng-configfilters/flume-ng-config-filter-api/pom.xml b/flume-ng-configfilters/flume-ng-config-filter-api/pom.xml
index 839ffb95f..18c8b0d4e 100644
--- a/flume-ng-configfilters/flume-ng-config-filter-api/pom.xml
+++ b/flume-ng-configfilters/flume-ng-config-filter-api/pom.xml
@@ -20,7 +20,7 @@ limitations under the License.
<parent>
<artifactId>flume-ng-configfilters</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/flume-ng-configfilters/flume-ng-environment-variable-config-filter/pom.xml b/flume-ng-configfilters/flume-ng-environment-variable-config-filter/pom.xml
index 55fbaa60f..5fce7ede0 100644
--- a/flume-ng-configfilters/flume-ng-environment-variable-config-filter/pom.xml
+++ b/flume-ng-configfilters/flume-ng-environment-variable-config-filter/pom.xml
@@ -20,7 +20,7 @@ limitations under the License.
<parent>
<artifactId>flume-ng-configfilters</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/flume-ng-configfilters/flume-ng-external-process-config-filter/pom.xml b/flume-ng-configfilters/flume-ng-external-process-config-filter/pom.xml
index 940945991..64679ec34 100644
--- a/flume-ng-configfilters/flume-ng-external-process-config-filter/pom.xml
+++ b/flume-ng-configfilters/flume-ng-external-process-config-filter/pom.xml
@@ -21,7 +21,7 @@ limitations under the License.
<parent>
<artifactId>flume-ng-configfilters</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/flume-ng-configfilters/pom.xml b/flume-ng-configfilters/pom.xml
index 7cc35884a..6e7f99c18 100644
--- a/flume-ng-configfilters/pom.xml
+++ b/flume-ng-configfilters/pom.xml
@@ -20,7 +20,7 @@ limitations under the License.
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../flume-parent/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/flume-ng-configuration/pom.xml b/flume-ng-configuration/pom.xml
index 218e06657..edb67e188 100644
--- a/flume-ng-configuration/pom.xml
+++ b/flume-ng-configuration/pom.xml
@@ -20,7 +20,7 @@ limitations under the License.
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../flume-parent/pom.xml</relativePath>
</parent>
<artifactId>flume-ng-configuration</artifactId>
diff --git a/flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/SinkType.java b/flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/SinkType.java
index 645dba0d0..aadffd9ac 100644
--- a/flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/SinkType.java
+++ b/flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/SinkType.java
@@ -54,7 +54,7 @@ public enum SinkType implements ComponentWithClassName {
/**
* HDFS Sink provided by org.apache.flume.sink.hdfs.HDFSEventSink
*/
- HDFS("org.apache.flume.sink.hdfs.HDFSEventSink"),
+ //HDFS("org.apache.flume.sink.hdfs.HDFSEventSink"),
/**
* IRC Sink provided by org.apache.flume.sink.irc.IRCSink
@@ -66,28 +66,28 @@ public enum SinkType implements ComponentWithClassName {
*
* @see AvroSink
*/
- AVRO("org.apache.flume.sink.AvroSink"),
+ AVRO("org.apache.flume.rpc.avro.sink.AvroSink"),
/**
* Thrift sink
*
* @see ThriftSink
*/
- THRIFT("org.apache.flume.sink.ThriftSink"),
+ THRIFT("org.apache.flume.rpc.thrift.sink.ThriftSink"),
/**
* ElasticSearch sink
*
* @see org.apache.flume.sink.elasticsearch.ElasticSearchSink
*/
- ELASTICSEARCH("org.apache.flume.sink.elasticsearch.ElasticSearchSink"),
+ //ELASTICSEARCH("org.apache.flume.sink.elasticsearch.ElasticSearchSink"),
/**
* HBase sink
*
* @see org.apache.flume.sink.hbase.HBaseSink
*/
- HBASE("org.apache.flume.sink.hbase.HBaseSink"),
+ //HBASE("org.apache.flume.sink.hbase.HBaseSink"),
/**
* AsyncHBase sink
@@ -101,20 +101,13 @@ public enum SinkType implements ComponentWithClassName {
*
* @see org.apache.flume.sink.hbase2.HBase2Sink
*/
- HBASE2("org.apache.flume.sink.hbase2.HBase2Sink"),
-
- /**
- * MorphlineSolr sink
- *
- * @see org.apache.flume.sink.solr.morphline.MorphlineSolrSink
- */
- MORPHLINE_SOLR("org.apache.flume.sink.solr.morphline.MorphlineSolrSink"),
+ //HBASE2("org.apache.flume.sink.hbase2.HBase2Sink"),
/**
* Hive Sink
* @see org.apache.flume.sink.hive.HiveSink
*/
- HIVE("org.apache.flume.sink.hive.HiveSink"),
+ //HIVE("org.apache.flume.sink.hive.HiveSink"),
/**
* HTTP Sink
diff --git a/flume-ng-configuration/src/main/java/org/apache/flume/conf/source/SourceType.java b/flume-ng-configuration/src/main/java/org/apache/flume/conf/source/SourceType.java
index b556b6c67..68f93d20b 100644
--- a/flume-ng-configuration/src/main/java/org/apache/flume/conf/source/SourceType.java
+++ b/flume-ng-configuration/src/main/java/org/apache/flume/conf/source/SourceType.java
@@ -56,7 +56,7 @@ public enum SourceType implements ComponentWithClassName {
*
* @see org.apache.flume.source.AvroSource
*/
- AVRO("org.apache.flume.source.AvroSource"),
+ AVRO("org.apache.flume.rpc.avro.source.AvroSource"),
/**
* SyslogTcpSource
@@ -84,7 +84,7 @@ public enum SourceType implements ComponentWithClassName {
*
* @see org.apache.flume.source.SpoolDirectorySource
*/
- SPOOLDIR("org.apache.flume.source.SpoolDirectorySource"),
+ SPOOLDIR("org.apache.flume.rpc.avro.source.SpoolDirectorySource"),
/**
* HTTP Source
@@ -98,7 +98,7 @@ public enum SourceType implements ComponentWithClassName {
*
* @see org.apache.flume.source.ThriftSource
*/
- THRIFT("org.apache.flume.source.ThriftSource"),
+ THRIFT("org.apache.flume.rpc.thrift.source.ThriftSource"),
/**
* JMS Source
diff --git a/flume-ng-core/pom.xml b/flume-ng-core/pom.xml
index 2ea97b734..2d2873bbf 100644
--- a/flume-ng-core/pom.xml
+++ b/flume-ng-core/pom.xml
@@ -22,7 +22,7 @@ limitations under the License.
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../flume-parent/pom.xml</relativePath>
</parent>
@@ -36,99 +36,6 @@ limitations under the License.
<pmd.maxAllowedViolations>121</pmd.maxAllowedViolations>
<module.name>org.apache.flume.core</module.name>
</properties>
-
- <build>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.eclipse.m2e</groupId>
- <artifactId>lifecycle-mapping</artifactId>
- <version>${lifecycle-mapping.version}</version>
- <configuration>
- <lifecycleMappingMetadata>
- <pluginExecutions>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-maven-plugin</artifactId>
- <versionRange>[${avro.version},)</versionRange>
- <goals>
- <goal>idl-protocol</goal>
- <goal>schema</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <execute />
- </action>
- </pluginExecution>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer-maven-plugin</artifactId>
- <versionRange>[${mvn-paranamer-plugin.version},)</versionRange>
- <goals>
- <goal>generate</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <ignore />
- </action>
- </pluginExecution>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <versionRange>[${mvn-antrun-plugin.version},)</versionRange>
- <goals>
- <goal>run</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <ignore />
- </action>
- </pluginExecution>
- </pluginExecutions>
- </lifecycleMappingMetadata>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
-
- <plugins>
-
- <plugin>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-maven-plugin</artifactId>
- <executions>
- <execution>
- <phase>generate-sources</phase>
- <goals>
- <goal>schema</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>run</id>
- <configuration>
- <sourceDirectory>${project.build.directory}/generated-sources/avro</sourceDirectory>
- <outputDirectory>${project.build.directory}/classes</outputDirectory>
- </configuration>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
<profiles>
<profile>
<id>not-windows</id>
@@ -336,26 +243,11 @@ limitations under the License.
<artifactId>commons-lang</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-ipc-netty</artifactId>
- </dependency>
-
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
-
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
@@ -399,11 +291,6 @@ limitations under the License.
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.mina</groupId>
<artifactId>mina-core</artifactId>
diff --git a/flume-ng-core/src/main/java/org/apache/flume/Channel.java b/flume-ng-core/src/main/java/org/apache/flume/Channel.java
index f1ae724fe..4db2f89bc 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/Channel.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/Channel.java
@@ -19,6 +19,7 @@ package org.apache.flume;
import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.lifecycle.LifecycleAware;
/**
diff --git a/flume-ng-core/src/main/java/org/apache/flume/channel/BasicChannelSemantics.java b/flume-ng-core/src/main/java/org/apache/flume/channel/BasicChannelSemantics.java
index 2b2506d87..3fa19453c 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/channel/BasicChannelSemantics.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/channel/BasicChannelSemantics.java
@@ -20,7 +20,7 @@
package org.apache.flume.channel;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Event;
import org.apache.flume.Transaction;
import org.apache.flume.annotations.InterfaceAudience;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/channel/BasicTransactionSemantics.java b/flume-ng-core/src/main/java/org/apache/flume/channel/BasicTransactionSemantics.java
index 5d99518a6..ddd6b9fd4 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/channel/BasicTransactionSemantics.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/channel/BasicTransactionSemantics.java
@@ -20,7 +20,7 @@
package org.apache.flume.channel;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Event;
import org.apache.flume.Transaction;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelProcessor.java b/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelProcessor.java
index 69878609c..5fb202a8d 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelProcessor.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelProcessor.java
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.Map;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
import org.apache.flume.ChannelSelector;
import org.apache.flume.Context;
import org.apache.flume.Event;
@@ -36,6 +35,7 @@ import org.apache.flume.interceptor.Interceptor;
import org.apache.flume.interceptor.InterceptorChain;
import org.apache.flume.Transaction;
import org.apache.flume.conf.Configurable;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.interceptor.InterceptorBuilderFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelUtils.java b/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelUtils.java
index 42137dd89..3fa7f288b 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelUtils.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelUtils.java
@@ -26,7 +26,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Event;
import org.apache.flume.Transaction;
import org.slf4j.Logger;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/channel/MemoryChannel.java b/flume-ng-core/src/main/java/org/apache/flume/channel/MemoryChannel.java
index 20da5721b..fff91c02a 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/channel/MemoryChannel.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/channel/MemoryChannel.java
@@ -20,8 +20,8 @@ package org.apache.flume.channel;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.flume.ChannelException;
-import org.apache.flume.ChannelFullException;
+import org.apache.flume.exception.ChannelException;
+import org.apache.flume.exception.ChannelFullException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.annotations.InterfaceAudience;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/channel/PseudoTxnMemoryChannel.java b/flume-ng-core/src/main/java/org/apache/flume/channel/PseudoTxnMemoryChannel.java
index 22e100238..8bce3bbf0 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/channel/PseudoTxnMemoryChannel.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/channel/PseudoTxnMemoryChannel.java
@@ -22,7 +22,7 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.Transaction;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/formatter/output/RollTimePathManager.java b/flume-ng-core/src/main/java/org/apache/flume/formatter/output/RollTimePathManager.java
index b14a812fa..28baa4c2c 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/formatter/output/RollTimePathManager.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/formatter/output/RollTimePathManager.java
@@ -20,18 +20,17 @@
package org.apache.flume.formatter.output;
import org.apache.flume.Context;
-import org.joda.time.LocalDateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
import java.io.File;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
/**
*
*/
public class RollTimePathManager extends DefaultPathManager {
- private final DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyyMMddHHmmss");
+ private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
private String lastRoll;
public RollTimePathManager(Context context) {
@@ -41,7 +40,7 @@ public class RollTimePathManager extends DefaultPathManager {
@Override
public File nextFile() {
StringBuilder sb = new StringBuilder();
- String date = formatter.print(LocalDateTime.now());
+ String date = formatter.format(LocalDateTime.now());
if (!date.equals(lastRoll)) {
getFileIndex().set(0);
lastRoll = date;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/interceptor/RegexExtractorInterceptorMillisSerializer.java b/flume-ng-core/src/main/java/org/apache/flume/interceptor/RegexExtractorInterceptorMillisSerializer.java
index 83bf0c930..81d6d3de9 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/interceptor/RegexExtractorInterceptorMillisSerializer.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/interceptor/RegexExtractorInterceptorMillisSerializer.java
@@ -20,9 +20,9 @@ package org.apache.flume.interceptor;
import org.apache.commons.lang.StringUtils;
import org.apache.flume.Context;
import org.apache.flume.conf.ComponentConfiguration;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import com.google.common.base.Preconditions;
@@ -40,13 +40,13 @@ public class RegexExtractorInterceptorMillisSerializer implements
String pattern = context.getString("pattern");
Preconditions.checkArgument(!StringUtils.isEmpty(pattern),
"Must configure with a valid pattern");
- formatter = DateTimeFormat.forPattern(pattern);
+ formatter = DateTimeFormatter.ofPattern(pattern);
}
@Override
public String serialize(String value) {
- DateTime dateTime = formatter.parseDateTime(value);
- return Long.toString(dateTime.getMillis());
+ LocalDateTime dateTime = LocalDateTime.parse(value, formatter);
+ return Long.toString(dateTime.toInstant(ZoneOffset.UTC).toEpochMilli());
}
@Override
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializerType.java b/flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializerType.java
index ce181308f..7606f4a00 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializerType.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializerType.java
@@ -25,15 +25,31 @@ import org.apache.flume.annotations.InterfaceStability;
@InterfaceStability.Unstable
public enum EventDeserializerType {
LINE(LineDeserializer.Builder.class),
- AVRO(AvroEventDeserializer.Builder.class),
- OTHER(null);
+ AVRO("org.apache.flume.rpc.avro.serialization.AvroDeserializer$Builder"),
+ OTHER();
private final Class<? extends EventDeserializer.Builder> builderClass;
+ EventDeserializerType() {
+ this.builderClass = null;
+ }
+
EventDeserializerType(Class<? extends EventDeserializer.Builder> builderClass) {
this.builderClass = builderClass;
}
+ @SuppressWarnings("unchecked")
+ EventDeserializerType(String builderClassName) {
+ Class<? extends EventDeserializer.Builder> clazz;
+
+ try {
+ clazz = (Class<? extends EventDeserializer.Builder>)Class.forName(builderClassName);
+ } catch (ClassNotFoundException ex) {
+ clazz= null;
+ }
+ this.builderClass = clazz;
+ }
+
public Class<? extends EventDeserializer.Builder> getBuilderClass() {
return builderClass;
}
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializerType.java b/flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializerType.java
index 86d243fe1..81948942d 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializerType.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializerType.java
@@ -26,15 +26,31 @@ import org.apache.flume.annotations.InterfaceStability;
public enum EventSerializerType {
TEXT(BodyTextEventSerializer.Builder.class),
HEADER_AND_TEXT(HeaderAndBodyTextEventSerializer.Builder.class),
- AVRO_EVENT(FlumeEventAvroEventSerializer.Builder.class),
- OTHER(null);
+ AVRO_EVENT("org.apache.flume.roc.avro.serialization.AvroEventSerializer$Builder"),
+ OTHER();
private final Class<? extends EventSerializer.Builder> builderClass;
+ EventSerializerType() {
+ this.builderClass = null;
+ }
+
EventSerializerType(Class<? extends EventSerializer.Builder> builderClass) {
this.builderClass = builderClass;
}
+ @SuppressWarnings("unchecked")
+ EventSerializerType(String builderClassName) {
+ Class<? extends EventSerializer.Builder> clazz;
+
+ try {
+ clazz = (Class<? extends EventSerializer.Builder>)Class.forName(builderClassName);
+ } catch (ClassNotFoundException ex) {
+ clazz = null;
+ }
+ this.builderClass = clazz;
+ }
+
public Class<? extends EventSerializer.Builder> getBuilderClass() {
return builderClass;
}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/serialization/TransientPositionTracker.java b/flume-ng-core/src/main/java/org/apache/flume/serialization/TransientPositionTracker.java
similarity index 100%
rename from flume-ng-core/src/test/java/org/apache/flume/serialization/TransientPositionTracker.java
rename to flume-ng-core/src/main/java/org/apache/flume/serialization/TransientPositionTracker.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractRpcSink.java b/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractRpcSink.java
index 12e2874e0..7414023b2 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractRpcSink.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractRpcSink.java
@@ -23,7 +23,6 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
@@ -34,6 +33,7 @@ import org.apache.flume.api.RpcClient;
import org.apache.flume.api.RpcClientConfigurationConstants;
import org.apache.flume.conf.BatchSizeSupported;
import org.apache.flume.conf.Configurable;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.instrumentation.SinkCounter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -405,7 +405,7 @@ public abstract class AbstractRpcSink extends AbstractSink implements Configurab
}
@VisibleForTesting
- RpcClient getUnderlyingClient() {
+ public RpcClient getUnderlyingClient() {
return client;
}
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/NetcatSource.java b/flume-ng-core/src/main/java/org/apache/flume/source/NetcatSource.java
index 67e7e481e..16bbade98 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/source/NetcatSource.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/source/NetcatSource.java
@@ -35,7 +35,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.CounterGroup;
import org.apache.flume.Event;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/NetcatUdpSource.java b/flume-ng-core/src/main/java/org/apache/flume/source/NetcatUdpSource.java
index 23ef8e3e7..b92adac26 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/source/NetcatUdpSource.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/source/NetcatUdpSource.java
@@ -25,7 +25,7 @@ import java.util.HashMap;
import java.util.Map;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.CounterGroup;
import org.apache.flume.Event;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/SequenceGeneratorSource.java b/flume-ng-core/src/main/java/org/apache/flume/source/SequenceGeneratorSource.java
index 6719606da..c1998a4fc 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/source/SequenceGeneratorSource.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/source/SequenceGeneratorSource.java
@@ -20,7 +20,7 @@
package org.apache.flume.source;
import com.google.common.base.Preconditions;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/StressSource.java b/flume-ng-core/src/main/java/org/apache/flume/source/StressSource.java
index 6c3c3304e..3e832ac23 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/source/StressSource.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/source/StressSource.java
@@ -24,7 +24,7 @@ import java.util.Arrays;
import java.util.List;
import org.apache.flume.source.shaded.guava.RateLimiter;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.CounterGroup;
import org.apache.flume.Event;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java
index f1db1553d..6bd580b03 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java
@@ -28,6 +28,14 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.collect.Maps;
import java.nio.charset.Charset;
+import java.time.LocalDateTime;
+import java.time.Year;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.ChronoField;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
@@ -36,9 +44,6 @@ import org.apache.flume.Event;
import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
import org.apache.flume.event.EventBuilder;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,28 +56,44 @@ public class SyslogParser {
private static final int TS_CACHE_MAX = 1000; // timestamp cache size limit
private static final Pattern TWO_SPACES = Pattern.compile(" ");
- private static final DateTimeFormatter rfc3164Format =
- DateTimeFormat.forPattern("MMM d HH:mm:ss").withZoneUTC();
+ private static final DateTimeFormatter RFC3164_FORMAT =
+ new DateTimeFormatterBuilder().appendPattern("MMM d HH:mm:ss")
+ .parseDefaulting(ChronoField.YEAR, Year.now().getValue()) // Adds current year
+ .toFormatter();
+ private static final DateTimeFormatter RFC5424_FORMAT =
+ new DateTimeFormatterBuilder()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE_TIME) // Parses YYYY-MM-DDTHH:mm:ss.SSSSSS
+ .optionalStart() // Start of optional timezone section
+ .appendOffset("+HH:MM", "Z") // Handles +00:00, -05:00, or Z
+ .optionalEnd() // End of optional timezone section
+ .toFormatter();
private static final String timePat = "yyyy-MM-dd'T'HH:mm:ss";
private static final int RFC3164_LEN = 15;
private static final int RFC5424_PREFIX_LEN = 19;
- private final DateTimeFormatter timeParser;
private LoadingCache<String, Long> timestampCache;
public SyslogParser() {
- timeParser = DateTimeFormat.forPattern(timePat).withZoneUTC();
timestampCache = CacheBuilder.newBuilder().maximumSize(TS_CACHE_MAX).build(
new CacheLoader<String, Long>() {
@Override
public Long load(String key) throws Exception {
- return timeParser.parseMillis(key);
+ return parseTime(key);
}
});
}
+ private long parseTime(String timestamp) {
+ char dateStartChar = timestamp.charAt(0);
+ if ((dateStartChar >= 'A' && dateStartChar <= 'Z')) {
+ return LocalDateTime.parse(timestamp, RFC3164_FORMAT).toInstant(ZoneOffset.UTC).toEpochMilli();
+ } else {
+ return ZonedDateTime.parse(timestamp, RFC5424_FORMAT.withZone(java.time.ZoneOffset.UTC)).toInstant().toEpochMilli();
+ }
+ }
+
/**
* Parses a Flume Event out of a syslog message string.
* @param msg Syslog message, not including the newline character
@@ -196,7 +217,6 @@ public class SyslogParser {
* @return Typical (for Java) milliseconds since UNIX epoch
*/
protected long parseRfc5424Date(String msg) {
-
Long ts = null;
int curPos = 0;
@@ -291,7 +311,6 @@ public class SyslogParser {
}
-
return ts;
}
@@ -303,15 +322,15 @@ public class SyslogParser {
* @return Typical (for Java) milliseconds since the UNIX epoch
*/
protected long parseRfc3164Time(String ts) {
- DateTime now = DateTime.now();
+ LocalDateTime now = LocalDateTime.now();
int year = now.getYear();
ts = TWO_SPACES.matcher(ts).replaceFirst(" ");
- DateTime date;
+ LocalDateTime date;
try {
- date = rfc3164Format.parseDateTime(ts);
- } catch (IllegalArgumentException e) {
+ date = LocalDateTime.parse(ts, RFC3164_FORMAT);
+ } catch (IllegalArgumentException | DateTimeParseException e) {
logger.debug("rfc3164 date parse failed on (" + ts + "): invalid format", e);
return 0;
}
@@ -328,7 +347,7 @@ public class SyslogParser {
*/
if (date != null) {
- DateTime fixed = date.withYear(year);
+ LocalDateTime fixed = date.withYear(year);
// flume clock is ahead or there is some latency, and the year rolled
if (fixed.isAfter(now) && fixed.minusMonths(1).isAfter(now)) {
@@ -344,7 +363,7 @@ public class SyslogParser {
return 0;
}
- return date.getMillis();
+ return date.toInstant(ZoneOffset.UTC).toEpochMilli();
}
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogTcpSource.java b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogTcpSource.java
index 366dbcd32..7972b8aee 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogTcpSource.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogTcpSource.java
@@ -27,7 +27,7 @@ import java.util.Set;
import javax.net.ssl.SSLEngine;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDrivenSource;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUDPSource.java b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUDPSource.java
index 2589855e3..4895e4796 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUDPSource.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUDPSource.java
@@ -25,7 +25,7 @@ import java.util.Set;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDrivenSource;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/http/HTTPSource.java b/flume-ng-core/src/main/java/org/apache/flume/source/http/HTTPSource.java
index d7d0433c8..bc25a020f 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/source/http/HTTPSource.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/source/http/HTTPSource.java
@@ -19,7 +19,7 @@ package org.apache.flume.source.http;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDrivenSource;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/channel/AbstractBasicChannelSemanticsTest.java b/flume-ng-core/src/test/java/org/apache/flume/channel/AbstractBasicChannelSemanticsTest.java
index 7600856e8..1b73fcb56 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/channel/AbstractBasicChannelSemanticsTest.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/channel/AbstractBasicChannelSemanticsTest.java
@@ -28,7 +28,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Event;
import org.apache.flume.event.EventBuilder;
import org.junit.After;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/channel/MockChannel.java b/flume-ng-core/src/test/java/org/apache/flume/channel/MockChannel.java
index 24b01e25e..ab84ff1df 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/channel/MockChannel.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/channel/MockChannel.java
@@ -22,7 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Event;
import org.apache.flume.Transaction;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/channel/TestChannelProcessor.java b/flume-ng-core/src/test/java/org/apache/flume/channel/TestChannelProcessor.java
index 93bc0cf51..488080088 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/channel/TestChannelProcessor.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/channel/TestChannelProcessor.java
@@ -21,7 +21,7 @@ package org.apache.flume.channel;
import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.ChannelSelector;
import org.apache.flume.Context;
import org.apache.flume.Event;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/channel/TestChannelUtils.java b/flume-ng-core/src/test/java/org/apache/flume/channel/TestChannelUtils.java
index 4cfa49119..a408fbb33 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/channel/TestChannelUtils.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/channel/TestChannelUtils.java
@@ -21,7 +21,7 @@ package org.apache.flume.channel;
import java.util.List;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Event;
import org.junit.Assert;
import org.junit.Test;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/channel/TestMemoryChannel.java b/flume-ng-core/src/test/java/org/apache/flume/channel/TestMemoryChannel.java
index f7e43eb01..461924072 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/channel/TestMemoryChannel.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/channel/TestMemoryChannel.java
@@ -21,7 +21,7 @@ package org.apache.flume.channel;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableMap;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/channel/TestMemoryChannelConcurrency.java b/flume-ng-core/src/test/java/org/apache/flume/channel/TestMemoryChannelConcurrency.java
index 68aa11751..8e12b3aa8 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/channel/TestMemoryChannelConcurrency.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/channel/TestMemoryChannelConcurrency.java
@@ -18,7 +18,7 @@
package org.apache.flume.channel;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.Transaction;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/formatter/output/TestBucketPath.java b/flume-ng-core/src/test/java/org/apache/flume/formatter/output/TestBucketPath.java
index 989c2a4a7..f7981d452 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/formatter/output/TestBucketPath.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/formatter/output/TestBucketPath.java
@@ -19,8 +19,10 @@
package org.apache.flume.formatter.output;
import org.apache.flume.Clock;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
+
+import java.time.Instant;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -36,6 +38,7 @@ import java.util.Map;
import java.util.SimpleTimeZone;
import java.util.TimeZone;
+import static java.time.format.DateTimeFormatter.ISO_DATE_TIME;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -252,9 +255,8 @@ public class TestBucketPath {
@Test
public void testDateRace() {
Clock mockClock = mock(Clock.class);
- DateTimeFormatter parser = ISODateTimeFormat.dateTimeParser();
- long two = parser.parseMillis("2013-04-21T02:59:59-00:00");
- long three = parser.parseMillis("2013-04-21T03:00:00-00:00");
+ long two = ZonedDateTime.parse("2013-04-21T02:59:59-00:00", ISO_DATE_TIME).toInstant().toEpochMilli();
+ long three = ZonedDateTime.parse("2013-04-21T03:00:00-00:00", ISO_DATE_TIME).toInstant().toEpochMilli();
when(mockClock.currentTimeMillis()).thenReturn(two, three);
// save & modify static state (yuck)
diff --git a/flume-ng-core/src/test/java/org/apache/flume/interceptor/TestRegexExtractorInterceptor.java b/flume-ng-core/src/test/java/org/apache/flume/interceptor/TestRegexExtractorInterceptor.java
index f75d31593..46e66dfc3 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/interceptor/TestRegexExtractorInterceptor.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/interceptor/TestRegexExtractorInterceptor.java
@@ -17,12 +17,15 @@
*/
package org.apache.flume.interceptor;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.event.EventBuilder;
import org.apache.flume.interceptor.Interceptor.Builder;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -30,6 +33,8 @@ import org.junit.Test;
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
+import static java.time.ZoneOffset.UTC;
+
public class TestRegexExtractorInterceptor {
private Builder fixtureBuilder;
@@ -163,8 +168,10 @@ public class TestRegexExtractorInterceptor {
throws Exception {
long now = (System.currentTimeMillis() / 60000L) * 60000L;
String pattern = "yyyy-MM-dd HH:mm:ss,SSS";
- DateTimeFormatter formatter = DateTimeFormat.forPattern(pattern);
- String body = formatter.print(now);
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern);
+ LocalDateTime current = Instant.ofEpochMilli(now).atOffset(UTC)
+ .toLocalDateTime();
+ String body = formatter.format(current);
System.out.println(body);
Context context = new Context();
// Skip the second group
diff --git a/flume-ng-core/src/test/java/org/apache/flume/interceptor/TestRegexExtractorInterceptorMillisSerializer.java b/flume-ng-core/src/test/java/org/apache/flume/interceptor/TestRegexExtractorInterceptorMillisSerializer.java
index dd42079a3..3d400e391 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/interceptor/TestRegexExtractorInterceptorMillisSerializer.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/interceptor/TestRegexExtractorInterceptorMillisSerializer.java
@@ -19,10 +19,14 @@ package org.apache.flume.interceptor;
import junit.framework.Assert;
import org.apache.flume.Context;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
import org.junit.Test;
+import static java.time.ZoneOffset.UTC;
+
public class TestRegexExtractorInterceptorMillisSerializer {
@Test
@@ -60,9 +64,11 @@ public class TestRegexExtractorInterceptorMillisSerializer {
context.put("pattern", pattern);
fixture.configure(context);
- DateTimeFormatter formatter = DateTimeFormat.forPattern(pattern);
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern);
long time = (System.currentTimeMillis() / 1000L) * 1000L;
+ LocalDateTime current = Instant.ofEpochMilli(time).atOffset(UTC)
+ .toLocalDateTime();
Assert.assertEquals(String.valueOf(time),
- fixture.serialize(formatter.print(time)));
+ fixture.serialize(formatter.format(current)));
}
}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/sink/TestDefaultSinkFactory.java b/flume-ng-core/src/test/java/org/apache/flume/sink/TestDefaultSinkFactory.java
index cf6cbbcc7..d34cb0624 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/sink/TestDefaultSinkFactory.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/sink/TestDefaultSinkFactory.java
@@ -37,20 +37,20 @@ public class TestDefaultSinkFactory {
@Test
public void testDuplicateCreate() {
- Sink avroSink1 = sinkFactory.create("avroSink1", "avro");
- Sink avroSink2 = sinkFactory.create("avroSink2", "avro");
+ Sink rollingSink1 = sinkFactory.create("rollingSink1", "file_roll");
+ Sink rollingSink2 = sinkFactory.create("rollingSink2", "file_roll");
- Assert.assertNotNull(avroSink1);
- Assert.assertNotNull(avroSink2);
- Assert.assertNotSame(avroSink1, avroSink2);
- Assert.assertTrue(avroSink1 instanceof AvroSink);
- Assert.assertTrue(avroSink2 instanceof AvroSink);
+ Assert.assertNotNull(rollingSink1);
+ Assert.assertNotNull(rollingSink2);
+ Assert.assertNotSame(rollingSink1, rollingSink2);
+ Assert.assertTrue(rollingSink1 instanceof RollingFileSink);
+ Assert.assertTrue(rollingSink2 instanceof RollingFileSink);
- Sink s1 = sinkFactory.create("avroSink1", "avro");
- Sink s2 = sinkFactory.create("avroSink2", "avro");
+ Sink s1 = sinkFactory.create("rollingSink1", "file_roll");
+ Sink s2 = sinkFactory.create("rollingSink2", "file_roll");
- Assert.assertNotSame(avroSink1, s1);
- Assert.assertNotSame(avroSink2, s2);
+ Assert.assertNotSame(rollingSink1, s1);
+ Assert.assertNotSame(rollingSink2, s2);
}
private void verifySinkCreation(String name, String type, Class<?> typeClass)
@@ -65,7 +65,7 @@ public class TestDefaultSinkFactory {
verifySinkCreation("null-sink", "null", NullSink.class);
verifySinkCreation("logger-sink", "logger", LoggerSink.class);
verifySinkCreation("file-roll-sink", "file_roll", RollingFileSink.class);
- verifySinkCreation("avro-sink", "avro", AvroSink.class);
+ //verifySinkCreation("avro-sink", "avro", AvroSink.class);
}
}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/sink/TestLoadBalancingSinkProcessor.java b/flume-ng-core/src/test/java/org/apache/flume/sink/TestLoadBalancingSinkProcessor.java
index 011d2d1f5..8c80ae4ae 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/sink/TestLoadBalancingSinkProcessor.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/sink/TestLoadBalancingSinkProcessor.java
@@ -19,7 +19,7 @@ package org.apache.flume.sink;
import junit.framework.Assert;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/sink/TestRollingFileSink.java b/flume-ng-core/src/test/java/org/apache/flume/sink/TestRollingFileSink.java
index c88b54165..9e0b5d593 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/sink/TestRollingFileSink.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/sink/TestRollingFileSink.java
@@ -20,7 +20,7 @@
package org.apache.flume.sink;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestAvroSource.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestAvroSource.java
deleted file mode 100644
index 7cd2dc07f..000000000
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestAvroSource.java
+++ /dev/null
@@ -1,610 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.source;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.security.cert.X509Certificate;
-import java.nio.channels.ServerSocketChannel;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.function.Consumer;
-
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLEngine;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
-
-import org.apache.avro.AvroRuntimeException;
-import org.apache.avro.ipc.netty.NettyTransceiver;
-import org.apache.avro.ipc.specific.SpecificRequestor;
-import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
-import org.apache.flume.ChannelSelector;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.FlumeException;
-import org.apache.flume.Transaction;
-import org.apache.flume.channel.ChannelProcessor;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.channel.ReplicatingChannelSelector;
-import org.apache.flume.conf.Configurables;
-import org.apache.flume.instrumentation.SourceCounter;
-import org.apache.flume.lifecycle.LifecycleController;
-import org.apache.flume.lifecycle.LifecycleState;
-import org.apache.flume.source.avro.AvroFlumeEvent;
-import org.apache.flume.source.avro.AvroSourceProtocol;
-import org.apache.flume.source.avro.Status;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.mockito.internal.util.reflection.Whitebox;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.netty.channel.ChannelPipeline;
-import io.netty.handler.codec.compression.JZlibDecoder;
-import io.netty.handler.codec.compression.JZlibEncoder;
-import io.netty.handler.codec.compression.ZlibEncoder;
-import io.netty.handler.ssl.SslHandler;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyListOf;
-import static org.mockito.Mockito.doThrow;
-
-public class TestAvroSource {
-
- private static final Logger logger = LoggerFactory
- .getLogger(TestAvroSource.class);
-
- private int selectedPort;
- private AvroSource source;
- private Channel channel;
- private InetAddress localhost;
-
- @Before
- public void setUp() throws UnknownHostException {
- localhost = InetAddress.getByName("127.0.0.1");
- source = new AvroSource();
- channel = new MemoryChannel();
-
- Configurables.configure(channel, new Context());
-
- List<Channel> channels = new ArrayList<Channel>();
- channels.add(channel);
-
- ChannelSelector rcs = new ReplicatingChannelSelector();
- rcs.setChannels(channels);
-
- source.setChannelProcessor(new ChannelProcessor(rcs));
- }
-
- @Test
- public void testLifecycle() throws InterruptedException, IOException {
-
- Context context = new Context();
-
- context.put("port", String.valueOf(selectedPort = getFreePort()));
- context.put("bind", "0.0.0.0");
-
- Configurables.configure(source, context);
-
- source.start();
-
- Assert
- .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
- source, LifecycleState.START_OR_ERROR));
- Assert.assertEquals("Server is started", LifecycleState.START,
- source.getLifecycleState());
-
- source.stop();
- Assert.assertTrue("Reached stop or error",
- LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
- Assert.assertEquals("Server is stopped", LifecycleState.STOP,
- source.getLifecycleState());
- }
-
- @Test
- public void testSourceStoppedOnFlumeExceptionIfPortUsed()
- throws InterruptedException, IOException {
- final String loopbackIPv4 = "127.0.0.1";
- final int port = 10500;
-
- // create a dummy socket bound to a known port.
- try (ServerSocketChannel dummyServerSocket = ServerSocketChannel.open()) {
- dummyServerSocket.socket().setReuseAddress(true);
- dummyServerSocket.socket().bind(new InetSocketAddress(loopbackIPv4, port));
-
- Context context = new Context();
- context.put("port", String.valueOf(port));
- context.put("bind", loopbackIPv4);
- Configurables.configure(source, context);
- try {
- source.start();
- Assert.fail("Expected an exception during startup caused by binding on a used port");
- } catch (FlumeException e) {
- logger.info("Received an expected exception.", e);
- Assert.assertTrue("Expected a server socket setup related root cause",
- e.getMessage().contains("server socket"));
- }
- }
- // As port is already in use, an exception is thrown and the source is stopped
- // cleaning up the opened sockets during source.start().
- Assert.assertEquals("Server is stopped", LifecycleState.STOP,
- source.getLifecycleState());
- }
-
- @Test
- public void testInvalidAddress()
- throws InterruptedException, IOException {
- final String invalidHost = "invalid.host";
- final int port = 10501;
-
- Context context = new Context();
- context.put("port", String.valueOf(port));
- context.put("bind", invalidHost);
- Configurables.configure(source, context);
-
- try {
- source.start();
- Assert.fail("Expected an exception during startup caused by binding on a invalid host");
- } catch (FlumeException e) {
- logger.info("Received an expected exception.", e);
- Assert.assertTrue("Expected a server socket setup related root cause",
- e.getMessage().contains("server socket"));
- }
-
- // As port is already in use, an exception is thrown and the source is stopped
- // cleaning up the opened sockets during source.start().
- Assert.assertEquals("Server is stopped", LifecycleState.STOP,
- source.getLifecycleState());
- }
-
- @Test
- public void testRequestWithNoCompression() throws InterruptedException, IOException {
-
- doRequest(false, false, 6);
- }
-
- @Test
- public void testRequestWithCompressionOnClientAndServerOnLevel0()
- throws InterruptedException, IOException {
-
- doRequest(true, true, 0);
- }
-
- @Test
- public void testRequestWithCompressionOnClientAndServerOnLevel1()
- throws InterruptedException, IOException {
-
- doRequest(true, true, 1);
- }
-
- @Test
- public void testRequestWithCompressionOnClientAndServerOnLevel6()
- throws InterruptedException, IOException {
-
- doRequest(true, true, 6);
- }
-
- @Test
- public void testRequestWithCompressionOnClientAndServerOnLevel9()
- throws InterruptedException, IOException {
-
- doRequest(true, true, 9);
- }
-
- @Test(expected = org.apache.avro.AvroRuntimeException.class)
- public void testRequestWithCompressionOnServerOnly() throws InterruptedException, IOException {
- //This will fail because both client and server need compression on
- doRequest(true, false, 6);
- }
-
- @Test(expected = org.apache.avro.AvroRuntimeException.class)
- public void testRequestWithCompressionOnClientOnly() throws InterruptedException, IOException {
- //This will fail because both client and server need compression on
- doRequest(false, true, 6);
- }
-
- private void doRequest(boolean serverEnableCompression, boolean clientEnableCompression,
- int compressionLevel) throws InterruptedException, IOException {
-
- Context context = new Context();
- context.put("port", String.valueOf(selectedPort = getFreePort()));
- context.put("bind", "0.0.0.0");
- context.put("threads", "50");
- if (serverEnableCompression) {
- context.put("compression-type", "deflate");
- } else {
- context.put("compression-type", "none");
- }
-
- Configurables.configure(source, context);
-
- source.start();
-
- Assert
- .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
- source, LifecycleState.START_OR_ERROR));
- Assert.assertEquals("Server is started", LifecycleState.START,
- source.getLifecycleState());
-
- AvroSourceProtocol client;
- NettyTransceiver nettyTransceiver;
- if (clientEnableCompression) {
-
- nettyTransceiver = new NettyTransceiver(new InetSocketAddress(selectedPort),
- (ch) -> {
- ChannelPipeline pipeline = ch.pipeline();
- ZlibEncoder encoder = new JZlibEncoder(compressionLevel);
- pipeline.addFirst("deflater", encoder);
- pipeline.addFirst("inflater", new JZlibDecoder());
- });
- client = SpecificRequestor.getClient(AvroSourceProtocol.class, nettyTransceiver);
- } else {
- nettyTransceiver = new NettyTransceiver(new InetSocketAddress(selectedPort));
- client = SpecificRequestor.getClient(AvroSourceProtocol.class, nettyTransceiver);
- }
-
- AvroFlumeEvent avroEvent = new AvroFlumeEvent();
-
- avroEvent.setHeaders(new HashMap<>());
- avroEvent.setBody(ByteBuffer.wrap("Hello avro".getBytes()));
-
- Status status = client.append(avroEvent);
-
- Assert.assertEquals(Status.OK, status);
-
- Transaction transaction = channel.getTransaction();
- transaction.begin();
-
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertEquals("Channel contained our event", "Hello avro",
- new String(event.getBody()));
- transaction.commit();
- transaction.close();
-
- logger.debug("Round trip event:{}", event);
-
-
- nettyTransceiver.close();
- source.stop();
- Assert.assertTrue("Reached stop or error",
- LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
- Assert.assertEquals("Server is stopped", LifecycleState.STOP,
- source.getLifecycleState());
- }
-
- @Test
- public void testSslRequestWithComponentKeystore() throws InterruptedException, IOException {
-
- Context context = new Context();
-
- context.put("port", String.valueOf(selectedPort = getFreePort()));
- context.put("bind", "0.0.0.0");
- context.put("ssl", "true");
- context.put("keystore", "src/test/resources/server.p12");
- context.put("keystore-password", "password");
- context.put("keystore-type", "PKCS12");
-
- Configurables.configure(source, context);
-
- doSslRequest();
- }
-
- @Test
- public void testSslRequestWithGlobalKeystore() throws InterruptedException, IOException {
-
- System.setProperty("javax.net.ssl.keyStore", "src/test/resources/server.p12");
- System.setProperty("javax.net.ssl.keyStorePassword", "password");
- System.setProperty("javax.net.ssl.keyStoreType", "PKCS12");
-
- Context context = new Context();
-
- context.put("port", String.valueOf(selectedPort = getFreePort()));
- context.put("bind", "0.0.0.0");
- context.put("ssl", "true");
-
- Configurables.configure(source, context);
-
- doSslRequest();
-
- System.clearProperty("javax.net.ssl.keyStore");
- System.clearProperty("javax.net.ssl.keyStorePassword");
- }
-
- private void doSslRequest() throws InterruptedException, IOException {
- source.start();
-
- Assert
- .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
- source, LifecycleState.START_OR_ERROR));
- Assert.assertEquals("Server is started", LifecycleState.START,
- source.getLifecycleState());
-
- AvroSourceProtocol client = SpecificRequestor.getClient(
- AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(selectedPort),
- (ch) -> {
- ChannelPipeline pipeline = ch.pipeline();
- SSLEngine engine = createSSLEngine();
- engine.setUseClientMode(true);
- pipeline.addFirst("ssl", new SslHandler(engine));
- }));
-
- AvroFlumeEvent avroEvent = new AvroFlumeEvent();
-
- avroEvent.setHeaders(new HashMap<>());
- avroEvent.setBody(ByteBuffer.wrap("Hello avro ssl".getBytes()));
-
- Status status = client.append(avroEvent);
-
- Assert.assertEquals(Status.OK, status);
-
- Transaction transaction = channel.getTransaction();
- transaction.begin();
-
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertEquals("Channel contained our event", "Hello avro ssl",
- new String(event.getBody()));
- transaction.commit();
- transaction.close();
-
- logger.debug("Round trip event:{}", event);
-
- source.stop();
- Assert.assertTrue("Reached stop or error",
- LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
- Assert.assertEquals("Server is stopped", LifecycleState.STOP,
- source.getLifecycleState());
- }
-
- private SSLEngine createSSLEngine() {
- try {
- SSLContext sslContext = SSLContext.getInstance("TLS");
- sslContext.init(null, new TrustManager[]{new PermissiveTrustManager()}, null);
- SSLEngine sslEngine = sslContext.createSSLEngine();
- sslEngine.setUseClientMode(true);
- return sslEngine;
- } catch (Exception ex) {
- throw new RuntimeException("Cannot create SSL channel", ex);
- }
- }
-
- /**
- * Bogus trust manager accepting any certificate
- */
- private static class PermissiveTrustManager implements X509TrustManager {
- @Override
- public void checkClientTrusted(X509Certificate[] certs, String s) {
- // nothing
- }
-
- @Override
- public void checkServerTrusted(X509Certificate[] certs, String s) {
- // nothing
- }
-
- @Override
- public X509Certificate[] getAcceptedIssuers() {
- return new X509Certificate[0];
- }
- }
-
- @Test
- public void testValidIpFilterAllows()
- throws InterruptedException, IOException {
- doIpFilterTest(localhost, "allow:name:localhost,deny:ip:*", true, false);
- doIpFilterTest(localhost, "allow:ip:" + localhost.getHostAddress() +
- ",deny:ip:*", true, false);
- doIpFilterTest(localhost, "allow:ip:*", true, false);
- doIpFilterTest(localhost, "allow:ip:" +
- localhost.getHostAddress().substring(0, 3) +
- "*,deny:ip:*", true, false);
- doIpFilterTest(localhost, "allow:ip:127.0.0.2,allow:ip:" +
- localhost.getHostAddress().substring(0, 3) +
- "*,deny:ip:*", true, false);
- doIpFilterTest(localhost, "allow:name:localhost,deny:ip:*", true, true);
- doIpFilterTest(localhost, "allow:ip:*", true, true);
- }
-
- @Test
- public void testValidIpFilterDenys()
- throws InterruptedException, IOException {
- doIpFilterTest(localhost, "deny:ip:*", false, false);
- doIpFilterTest(localhost, "deny:name:localhost", false, false);
- doIpFilterTest(localhost, "deny:ip:" + localhost.getHostAddress() +
- ",allow:ip:*", false, false);
- doIpFilterTest(localhost, "deny:ip:*", false, false);
- doIpFilterTest(localhost, "allow:ip:45.2.2.2,deny:ip:*", false, false);
- doIpFilterTest(localhost, "deny:ip:" +
- localhost.getHostAddress().substring(0, 3) +
- "*,allow:ip:*", false, false);
- doIpFilterTest(localhost, "deny:ip:*", false, true);
- }
-
- @Test
- public void testInvalidIpFilter() throws InterruptedException, IOException {
- doIpFilterTest(localhost, "deny:ip:*", false, false);
- doIpFilterTest(localhost, "allow:name:localhost", true, false);
- doIpFilterTest(localhost, "deny:ip:127.0.0.2,allow:ip:*,deny:ip:" +
- localhost.getHostAddress(), true, false);
- doIpFilterTest(localhost, "deny:ip:" +
- localhost.getHostAddress().substring(0, 3) + "*,allow:ip:*",
- false, false);
-
- // Private lambda expression to check the received FlumeException within this test
- Consumer<Exception> exceptionChecker = (Exception ex) -> {
- logger.info("Received an expected exception", ex);
- //covers all ipFilter related exceptions
- Assert.assertTrue("Expected an ipFilterRules related exception",
- ex.getMessage().contains("ipFilter"));
- };
-
- try {
- doIpFilterTest(localhost, null, false, false);
- Assert.fail("The null ipFilterRules config should have thrown an exception.");
- } catch (FlumeException e) {
- exceptionChecker.accept(e);
- }
-
- try {
- doIpFilterTest(localhost, "", true, false);
- Assert.fail("The empty string ipFilterRules config should have thrown "
- + "an exception");
- } catch (FlumeException e) {
- exceptionChecker.accept(e);
- }
-
- try {
- doIpFilterTest(localhost, "homer:ip:45.4.23.1", true, false);
- Assert.fail("Bad ipFilterRules config should have thrown an exception.");
- } catch (FlumeException e) {
- exceptionChecker.accept(e);
- }
-
- try {
- doIpFilterTest(localhost, "allow:sleeps:45.4.23.1", true, false);
- Assert.fail("Bad ipFilterRules config should have thrown an exception.");
- } catch (FlumeException e) {
- exceptionChecker.accept(e);
- }
- }
-
- public void doIpFilterTest(InetAddress dest, String ruleDefinition,
- boolean eventShouldBeAllowed, boolean testWithSSL)
- throws InterruptedException, IOException {
- Context context = new Context();
- context.put("port", String.valueOf(selectedPort = getFreePort()));
- context.put("bind", "0.0.0.0");
- context.put("ipFilter", "true");
- if (ruleDefinition != null) {
- context.put("ipFilterRules", ruleDefinition);
- }
- if (testWithSSL) {
- logger.info("Client testWithSSL" + testWithSSL);
- context.put("ssl", "true");
- context.put("keystore", "src/test/resources/server.p12");
- context.put("keystore-password", "password");
- context.put("keystore-type", "PKCS12");
- }
- // Invalid configuration may result in a FlumeException
- Configurables.configure(source, context);
- source.start();
-
- Assert
- .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
- source, LifecycleState.START_OR_ERROR));
- Assert.assertEquals("Server is started", LifecycleState.START,
- source.getLifecycleState());
-
- AvroSourceProtocol client;
- NettyTransceiver nettyTransceiver = null;
- try {
- if (testWithSSL) {
- nettyTransceiver = new NettyTransceiver(new InetSocketAddress(dest, selectedPort),
- (ch) -> {
- ChannelPipeline pipeline = ch.pipeline();
- SSLEngine engine = createSSLEngine();
- if (engine != null) {
- engine.setUseClientMode(true);
- pipeline.addFirst("ssl", new SslHandler(engine));
- }
- });
- client = SpecificRequestor.getClient(AvroSourceProtocol.class, nettyTransceiver);
- } else {
- nettyTransceiver = new NettyTransceiver(new InetSocketAddress(dest, selectedPort));
- client = SpecificRequestor.getClient(AvroSourceProtocol.class, nettyTransceiver);
- }
-
- AvroFlumeEvent avroEvent = new AvroFlumeEvent();
- avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
- avroEvent.setBody(ByteBuffer.wrap("Hello avro ipFilter".getBytes()));
-
- logger.info("Client about to append");
- Status status = client.append(avroEvent);
- logger.info("Client appended");
- Assert.assertEquals(Status.OK, status);
- } catch (IOException | AvroRuntimeException e) {
- Assert.assertTrue("Should have been allowed: " + ruleDefinition, !eventShouldBeAllowed);
- return;
- } finally {
- if (nettyTransceiver != null) {
- nettyTransceiver.close();
- }
- source.stop();
- }
- Assert.assertTrue("Should have been denied: " + ruleDefinition,
- eventShouldBeAllowed);
-
- Transaction transaction = channel.getTransaction();
- transaction.begin();
-
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertEquals("Channel contained our event", "Hello avro ipFilter",
- new String(event.getBody()));
- transaction.commit();
- transaction.close();
- logger.debug("Round trip event:{}", event);
-
- Assert.assertTrue("Reached stop or error",
- LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
- Assert.assertEquals("Server is stopped", LifecycleState.STOP,
- source.getLifecycleState());
- }
-
- @Test
- public void testErrorCounterChannelWriteFail() throws Exception {
- Context context = new Context();
- context.put("port", String.valueOf(selectedPort = getFreePort()));
- context.put("bind", "0.0.0.0");
- source.configure(context);
- ChannelProcessor cp = Mockito.mock(ChannelProcessor.class);
- doThrow(new ChannelException("dummy")).when(cp).processEvent(any(Event.class));
- doThrow(new ChannelException("dummy")).when(cp).processEventBatch(anyListOf(Event.class));
- source.setChannelProcessor(cp);
- source.start();
- AvroFlumeEvent avroEvent = new AvroFlumeEvent();
- avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
- avroEvent.setBody(ByteBuffer.wrap("Hello avro ssl".getBytes()));
- source.append(avroEvent);
- source.appendBatch(Arrays.asList(avroEvent));
- SourceCounter sc = (SourceCounter) Whitebox.getInternalState(source, "sourceCounter");
- Assert.assertEquals(2, sc.getChannelWriteFail());
- source.stop();
- }
-
- private static int getFreePort() throws IOException {
- try (ServerSocket socket = new ServerSocket(0)) {
- return socket.getLocalPort();
- }
- }
-
-
-}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestDefaultSourceFactory.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestDefaultSourceFactory.java
index 97e024f87..19d079a31 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestDefaultSourceFactory.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/source/TestDefaultSourceFactory.java
@@ -38,20 +38,20 @@ public class TestDefaultSourceFactory {
@Test
public void testDuplicateCreate() {
- Source avroSource1 = sourceFactory.create("avroSource1", "avro");
- Source avroSource2 = sourceFactory.create("avroSource2", "avro");
+ Source execSource1 = sourceFactory.create("execSource1", "exec");
+ Source execSource2 = sourceFactory.create("execSource2", "exec");
- Assert.assertNotNull(avroSource1);
- Assert.assertNotNull(avroSource2);
- Assert.assertNotSame(avroSource1, avroSource2);
- Assert.assertTrue(avroSource1 instanceof AvroSource);
- Assert.assertTrue(avroSource2 instanceof AvroSource);
+ Assert.assertNotNull(execSource1);
+ Assert.assertNotNull(execSource2);
+ Assert.assertNotSame(execSource1, execSource2);
+ Assert.assertTrue(execSource1 instanceof ExecSource);
+ Assert.assertTrue(execSource2 instanceof ExecSource);
- Source s1 = sourceFactory.create("avroSource1", "avro");
- Source s2 = sourceFactory.create("avroSource2", "avro");
+ Source s1 = sourceFactory.create("execSource1", "exec");
+ Source s2 = sourceFactory.create("execSource2", "exec");
- Assert.assertNotSame(avroSource1, s1);
- Assert.assertNotSame(avroSource2, s2);
+ Assert.assertNotSame(execSource1, s1);
+ Assert.assertNotSame(execSource2, s2);
}
@@ -68,15 +68,14 @@ public class TestDefaultSourceFactory {
verifySourceCreation("netcat-src", "netcat", NetcatSource.class);
verifySourceCreation("netcat-udp-src", "netcatudp", NetcatUdpSource.class);
verifySourceCreation("exec-src", "exec", ExecSource.class);
- verifySourceCreation("avro-src", "avro", AvroSource.class);
+ //verifySourceCreation("avro-src", "avro", AvroSource.class);
verifySourceCreation("syslogtcp-src", "syslogtcp", SyslogTcpSource.class);
verifySourceCreation("multiport_syslogtcp-src", "multiport_syslogtcp",
MultiportSyslogTCPSource.class);
verifySourceCreation("syslogudp-src", "syslogudp", SyslogUDPSource.class);
- verifySourceCreation("spooldir-src", "spooldir",
- SpoolDirectorySource.class);
+ //verifySourceCreation("spooldir-src", "spooldir", SpoolDirectorySource.class);
verifySourceCreation("http-src", "http", HTTPSource.class);
- verifySourceCreation("thrift-src", "thrift", ThriftSource.class);
+ //verifySourceCreation("thrift-src", "thrift", ThriftSource.class);
verifySourceCreation("custom-src", MockSource.class.getCanonicalName(),
MockSource.class);
}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestMultiportSyslogTCPSource.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestMultiportSyslogTCPSource.java
index c9c35e406..63e8100c0 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestMultiportSyslogTCPSource.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/source/TestMultiportSyslogTCPSource.java
@@ -32,6 +32,8 @@ import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.security.cert.X509Certificate;
import java.text.ParseException;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -41,7 +43,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.function.BiConsumer;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.ChannelSelector;
import org.apache.flume.Context;
import org.apache.flume.Event;
@@ -60,13 +62,13 @@ import org.apache.logging.log4j.Logger;
import org.apache.mina.core.buffer.IoBuffer;
import org.apache.mina.core.session.DefaultIoSessionDataStructureFactory;
import org.apache.mina.transport.socket.nio.NioSession;
-import org.joda.time.DateTime;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.internal.util.reflection.Whitebox;
+import static java.time.ZoneOffset.UTC;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -81,6 +83,7 @@ public class TestMultiportSyslogTCPSource {
private static final Logger LOGGER = LogManager.getLogger();
private static final String TEST_CLIENT_IP_HEADER = "testClientIPHeader";
private static final String TEST_CLIENT_HOSTNAME_HEADER = "testClientHostnameHeader";
+ private static final DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME;
private static final int getFreePort() throws IOException {
try (ServerSocket socket = new ServerSocket(0)) {
@@ -88,8 +91,8 @@ public class TestMultiportSyslogTCPSource {
}
}
- private final DateTime time = new DateTime();
- private final String stamp1 = time.toString();
+ private final ZonedDateTime time = ZonedDateTime.now();
+ private final String stamp1 = time.format(formatter);
private final String host1 = "localhost.localdomain";
private final String data1 = "proc1 - some msg";
@@ -297,7 +300,7 @@ public class TestMultiportSyslogTCPSource {
iter.remove();
Assert.assertEquals("Timestamps must match",
- String.valueOf(time.getMillis()), headers.get("timestamp"));
+ String.valueOf(time.toInstant().toEpochMilli()), headers.get("timestamp"));
String host2 = headers.get("host");
Assert.assertEquals(host1, host2);
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestNetcatSource.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestNetcatSource.java
index 7db8983fe..aa2c023c6 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestNetcatSource.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/source/TestNetcatSource.java
@@ -53,7 +53,7 @@ import java.util.List;
public class TestNetcatSource {
private static final Logger logger =
- LoggerFactory.getLogger(TestAvroSource.class);
+ LoggerFactory.getLogger(TestNetcatSource.class);
private static int getFreePort() {
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestSequenceGeneratorSource.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestSequenceGeneratorSource.java
index 473f94efb..a2a0832a2 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestSequenceGeneratorSource.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/source/TestSequenceGeneratorSource.java
@@ -18,7 +18,7 @@
*/
package org.apache.flume.source;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestStressSource.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestStressSource.java
index 043f4dca0..97fa7a31f 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestStressSource.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/source/TestStressSource.java
@@ -31,7 +31,7 @@ import java.util.List;
import junit.framework.Assert;
import junit.framework.TestCase;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.CounterGroup;
import org.apache.flume.Event;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogParser.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogParser.java
index 7edb9b737..ccc948527 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogParser.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogParser.java
@@ -21,8 +21,9 @@ package org.apache.flume.source;
import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import org.apache.flume.Event;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
+
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
import org.junit.Assert;
import org.junit.Test;
@@ -32,23 +33,43 @@ import java.util.List;
import java.util.Set;
public class TestSyslogParser {
+
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+
+ private static class Entry {
+ private final String formattedDate;
+ private final long millis;
+
+ public Entry(String formattedDate, long millis) {
+ this.formattedDate = formattedDate;
+ this.millis = millis;
+ }
+
+ public String getFormattedDate() {
+ return formattedDate;
+ }
+ public long getMillis() {
+ return millis;
+ }
+ }
+
@Test
public void testRfc5424DateParsing() {
- final String[] examples = {
- "1985-04-12T23:20:50.52Z", "1985-04-12T19:20:50.52-04:00",
- "2003-10-11T22:14:15.003Z", "2003-08-24T05:14:15.000003-07:00",
- "2012-04-13T11:11:11-08:00", "2012-04-13T08:08:08.0001+00:00",
- "2012-04-13T08:08:08.251+00:00"
- };
+ final Entry[] examples = {
+ new Entry("1985-04-12T23:20:50.52Z", 482196050520L),
+ new Entry("1985-04-12T19:20:50.52-04:00", 482196050520L),
+ new Entry("2003-10-11T22:14:15.003Z", 1065910455003L),
+ new Entry("2003-08-24T05:14:15.000003-07:00", 1061727255000L),
+ new Entry("2012-04-13T08:08:08.0001+00:00", 1334304488000L),
+ new Entry("2012-04-13T08:08:08.251+00:00", 1334304488251L)
+ };
SyslogParser parser = new SyslogParser();
- DateTimeFormatter jodaParser = ISODateTimeFormat.dateTimeParser();
+ //DateTimeFormatter timeParser = DateTimeFormatter
- for (String ex : examples) {
- Assert.assertEquals(
- "Problem parsing date string: " + ex,
- jodaParser.parseMillis(ex),
- parser.parseRfc5424Date(ex));
+ for (Entry entry : examples) {
+ long time = Instant.from(DateTimeFormatter.ISO_OFFSET_DATE_TIME.parse(entry.getFormattedDate())).toEpochMilli();
+ Assert.assertEquals(entry.millis + " is not the same as timestamp " + time, entry.getMillis(), time);
}
}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogTcpSource.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogTcpSource.java
index f8098d160..df4c760a5 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogTcpSource.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogTcpSource.java
@@ -20,7 +20,7 @@ package org.apache.flume.source;
import com.google.common.base.Charsets;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.ChannelSelector;
import org.apache.flume.Context;
import org.apache.flume.Event;
@@ -29,7 +29,6 @@ import org.apache.flume.channel.ChannelProcessor;
import org.apache.flume.channel.MemoryChannel;
import org.apache.flume.channel.ReplicatingChannelSelector;
import org.apache.flume.conf.Configurables;
-import org.joda.time.DateTime;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
@@ -41,6 +40,7 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.security.cert.X509Certificate;
+import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -65,7 +65,7 @@ public class TestSyslogTcpSource {
private SyslogTcpSource source;
private Channel channel;
private static final int TEST_SYSLOG_PORT = 0;
- private final DateTime time = new DateTime();
+ private final LocalDateTime time = LocalDateTime.now();
private final String stamp1 = time.toString();
private final String host1 = "localhost.localdomain";
private final String data1 = "test syslog data";
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogUdpSource.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogUdpSource.java
index 4b2c52b79..dbf4d0860 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogUdpSource.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogUdpSource.java
@@ -20,7 +20,7 @@ package org.apache.flume.source;
import com.google.common.base.Charsets;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.ChannelSelector;
import org.apache.flume.Context;
import org.apache.flume.Event;
@@ -29,10 +29,10 @@ import org.apache.flume.channel.ChannelProcessor;
import org.apache.flume.channel.MemoryChannel;
import org.apache.flume.channel.ReplicatingChannelSelector;
import org.apache.flume.conf.Configurables;
-import org.joda.time.DateTime;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
+import org.mockito.cglib.core.Local;
import org.slf4j.LoggerFactory;
import java.io.IOException;
@@ -40,6 +40,7 @@ import java.net.DatagramPacket;
import java.net.DatagramSocket;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -59,7 +60,7 @@ public class TestSyslogUdpSource {
private SyslogUDPSource source;
private Channel channel;
private static final int TEST_SYSLOG_PORT = 0;
- private final DateTime time = new DateTime();
+ private final LocalDateTime time = LocalDateTime.now();
private final String stamp1 = time.toString();
private final String host1 = "localhost.localdomain";
private final String data1 = "test syslog data";
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestThriftSource.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestThriftSource.java
deleted file mode 100644
index 28be4a1e1..000000000
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestThriftSource.java
+++ /dev/null
@@ -1,383 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.source;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
-import org.apache.flume.ChannelSelector;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.FlumeException;
-import org.apache.flume.Transaction;
-import org.apache.flume.api.RpcClient;
-import org.apache.flume.api.RpcClientConfigurationConstants;
-import org.apache.flume.api.RpcClientFactory;
-import org.apache.flume.channel.ChannelProcessor;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.channel.ReplicatingChannelSelector;
-import org.apache.flume.conf.Configurables;
-import org.apache.flume.event.EventBuilder;
-
-import org.apache.flume.instrumentation.SourceCounter;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.Ignore;
-import org.mockito.Mockito;
-import org.mockito.internal.util.reflection.Whitebox;
-
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyListOf;
-import static org.mockito.Mockito.doThrow;
-
-public class TestThriftSource {
-
- private ThriftSource source;
- private MemoryChannel channel;
- private RpcClient client;
- private final Properties props = new Properties();
- private int port;
-
- @Before
- public void setUp() throws IOException {
- try (ServerSocket socket = new ServerSocket(0)) {
- port = socket.getLocalPort();
- }
- props.clear();
- props.setProperty("hosts", "h1");
- props.setProperty("hosts.h1", "0.0.0.0:" + String.valueOf(port));
- props.setProperty(RpcClientConfigurationConstants.CONFIG_BATCH_SIZE, "10");
- props.setProperty(RpcClientConfigurationConstants.CONFIG_REQUEST_TIMEOUT, "2000");
- channel = new MemoryChannel();
- source = new ThriftSource();
- }
-
- @After
- public void stop() throws Exception {
- source.stop();
- }
-
- private void configureSource() {
- List<Channel> channels = new ArrayList<Channel>();
- channels.add(channel);
-
- ChannelSelector rcs = new ReplicatingChannelSelector();
- rcs.setChannels(channels);
-
- source.setChannelProcessor(new ChannelProcessor(rcs));
- }
-
- @Ignore("This test is flakey and causes tests to fail pretty often.")
- @Test
- public void testAppendSSLWithComponentKeystore() throws Exception {
-
- Context context = new Context();
- channel.configure(context);
- configureSource();
-
- context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
- context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
- context.put("ssl", "true");
- context.put("keystore", "src/test/resources/keystorefile.jks");
- context.put("keystore-password", "password");
- context.put("keystore-type", "JKS");
-
- Configurables.configure(source, context);
-
- doAppendSSL();
- }
-
- @Ignore("This test is flakey and causes tests to fail pretty often.")
- @Test
- public void testAppendSSLWithGlobalKeystore() throws Exception {
-
- System.setProperty("javax.net.ssl.keyStore", "src/test/resources/keystorefile.jks");
- System.setProperty("javax.net.ssl.keyStorePassword", "password");
- System.setProperty("javax.net.ssl.keyStoreType", "JKS");
-
- Context context = new Context();
- channel.configure(context);
- configureSource();
-
- context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
- context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
- context.put("ssl", "true");
-
- Configurables.configure(source, context);
-
- doAppendSSL();
-
- System.clearProperty("javax.net.ssl.keyStore");
- System.clearProperty("javax.net.ssl.keyStorePassword");
- System.clearProperty("javax.net.ssl.keyStoreType");
- }
-
- private void doAppendSSL() throws EventDeliveryException {
- Properties sslprops = (Properties)props.clone();
- sslprops.put("ssl", "true");
- sslprops.put("truststore", "src/test/resources/truststorefile.jks");
- sslprops.put("truststore-password", "password");
- client = RpcClientFactory.getThriftInstance(sslprops);
-
- source.start();
- for (int i = 0; i < 30; i++) {
- client.append(EventBuilder.withBody(String.valueOf(i).getBytes()));
- }
- Transaction transaction = channel.getTransaction();
- transaction.begin();
-
- for (int i = 0; i < 30; i++) {
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertEquals(String.valueOf(i), new String(event.getBody()));
- }
- transaction.commit();
- transaction.close();
- }
-
- @Test
- public void testAppend() throws Exception {
- client = RpcClientFactory.getThriftInstance(props);
- Context context = new Context();
- channel.configure(context);
- configureSource();
- context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
- context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
- Configurables.configure(source, context);
- source.start();
- for (int i = 0; i < 30; i++) {
- client.append(EventBuilder.withBody(String.valueOf(i).getBytes()));
- }
- Transaction transaction = channel.getTransaction();
- transaction.begin();
-
- for (int i = 0; i < 30; i++) {
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertEquals(String.valueOf(i), new String(event.getBody()));
- }
- transaction.commit();
- transaction.close();
- }
-
- @Test
- public void testAppendBatch() throws Exception {
- client = RpcClientFactory.getThriftInstance(props);
- Context context = new Context();
- context.put("capacity", "1000");
- context.put("transactionCapacity", "1000");
- channel.configure(context);
- configureSource();
- context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
- context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
- Configurables.configure(source, context);
- source.start();
- for (int i = 0; i < 30; i++) {
- List<Event> events = Lists.newArrayList();
- for (int j = 0; j < 10; j++) {
- Map<String, String> hdrs = Maps.newHashMap();
- hdrs.put("time", String.valueOf(System.currentTimeMillis()));
- events.add(EventBuilder.withBody(String.valueOf(i).getBytes(), hdrs));
- }
- client.appendBatch(events);
- }
- Transaction transaction = channel.getTransaction();
- transaction.begin();
- long after = System.currentTimeMillis();
- List<Integer> events = Lists.newArrayList();
- for (int i = 0; i < 300; i++) {
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertTrue(Long.valueOf(event.getHeaders().get("time")) <= after);
- events.add(Integer.parseInt(new String(event.getBody())));
- }
- transaction.commit();
- transaction.close();
-
- Collections.sort(events);
-
- int index = 0;
- //30 batches of 10
- for (int i = 0; i < 30; i++) {
- for (int j = 0; j < 10; j++) {
- Assert.assertEquals(i, events.get(index++).intValue());
- }
- }
- }
-
- @Test
- public void testAppendBigBatch() throws Exception {
- client = RpcClientFactory.getThriftInstance(props);
- Context context = new Context();
- context.put("capacity", "3000");
- context.put("transactionCapacity", "3000");
- channel.configure(context);
- configureSource();
- context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
- context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
- Configurables.configure(source, context);
- source.start();
- for (int i = 0; i < 5; i++) {
- List<Event> events = Lists.newArrayList();
- for (int j = 0; j < 500; j++) {
- Map<String, String> hdrs = Maps.newHashMap();
- hdrs.put("time", String.valueOf(System.currentTimeMillis()));
- events.add(EventBuilder.withBody(String.valueOf(i).getBytes(), hdrs));
- }
- client.appendBatch(events);
- }
- Transaction transaction = channel.getTransaction();
- transaction.begin();
- long after = System.currentTimeMillis();
- List<Integer> events = Lists.newArrayList();
- for (int i = 0; i < 2500; i++) {
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertTrue(Long.valueOf(event.getHeaders().get("time")) < after);
- events.add(Integer.parseInt(new String(event.getBody())));
- }
- transaction.commit();
- transaction.close();
-
- Collections.sort(events);
-
- int index = 0;
- //10 batches of 500
- for (int i = 0; i < 5; i++) {
- for (int j = 0; j < 500; j++) {
- Assert.assertEquals(i, events.get(index++).intValue());
- }
- }
- }
-
- @Test
- public void testMultipleClients() throws Exception {
- ExecutorService submitter = Executors.newCachedThreadPool();
- client = RpcClientFactory.getThriftInstance(props);
- Context context = new Context();
- context.put("capacity", "1000");
- context.put("transactionCapacity", "1000");
- channel.configure(context);
- configureSource();
- context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
- context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
- Configurables.configure(source, context);
- source.start();
- ExecutorCompletionService<Void> completionService = new ExecutorCompletionService<>(submitter);
- for (int i = 0; i < 30; i++) {
- completionService.submit(new SubmitHelper(i), null);
- }
- //wait for all threads to be done
-
-
- for (int i = 0; i < 30; i++) {
- completionService.take();
- }
-
- Transaction transaction = channel.getTransaction();
- transaction.begin();
- long after = System.currentTimeMillis();
- List<Integer> events = Lists.newArrayList();
- for (int i = 0; i < 300; i++) {
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertTrue(Long.valueOf(event.getHeaders().get("time")) < after);
- events.add(Integer.parseInt(new String(event.getBody())));
- }
- transaction.commit();
- transaction.close();
-
- Collections.sort(events);
-
- int index = 0;
- //30 batches of 10
- for (int i = 0; i < 30; i++) {
- for (int j = 0; j < 10; j++) {
- Assert.assertEquals(i, events.get(index++).intValue());
- }
- }
- }
-
- @Test
- public void testErrorCounterChannelWriteFail() throws Exception {
- client = RpcClientFactory.getThriftInstance(props);
- Context context = new Context();
- context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
- context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
- source.configure(context);
- ChannelProcessor cp = Mockito.mock(ChannelProcessor.class);
- doThrow(new ChannelException("dummy")).when(cp).processEvent(any(Event.class));
- doThrow(new ChannelException("dummy")).when(cp).processEventBatch(anyListOf(Event.class));
- source.setChannelProcessor(cp);
- source.start();
- Event event = EventBuilder.withBody("hello".getBytes());
- try {
- client.append(event);
- } catch (EventDeliveryException e) {
- //
- }
- try {
- client.appendBatch(Arrays.asList(event));
- } catch (EventDeliveryException e) {
- //
- }
- SourceCounter sc = (SourceCounter) Whitebox.getInternalState(source, "sourceCounter");
- Assert.assertEquals(2, sc.getChannelWriteFail());
- source.stop();
- }
-
- private class SubmitHelper implements Runnable {
- private final int i;
-
- public SubmitHelper(int i) {
- this.i = i;
- }
-
- @Override
- public void run() {
- List<Event> events = Lists.newArrayList();
- for (int j = 0; j < 10; j++) {
- Map<String, String> hdrs = Maps.newHashMap();
- hdrs.put("time", String.valueOf(System.currentTimeMillis()));
- events.add(EventBuilder.withBody(String.valueOf(i).getBytes(), hdrs));
- }
- try {
- client.appendBatch(events);
- } catch (EventDeliveryException e) {
- throw new FlumeException(e);
- }
- }
- }
-}
diff --git a/flume-ng-dist/pom.xml b/flume-ng-dist/pom.xml
index b2e400250..25478e4a4 100644
--- a/flume-ng-dist/pom.xml
+++ b/flume-ng-dist/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../flume-parent/pom.xml</relativePath>
</parent>
@@ -133,6 +133,14 @@
<artifactId>flume-kafka-channel</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-rpc-avro</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-rpc-thrift</artifactId>
+ </dependency>
+ <!--<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-hdfs-sink</artifactId>
</dependency>
@@ -151,39 +159,31 @@
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-morphline-solr-sink</artifactId>
- </dependency>
+ </dependency>-->
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-kafka-sink</artifactId>
</dependency>
- <dependency>
+ <!--<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-hive-sink</artifactId>
</dependency>
<dependency>
<groupId>org.apache.flume.flume-ng-sources</groupId>
<artifactId>flume-scribe-source</artifactId>
- </dependency>
+ </dependency>-->
<dependency>
<groupId>org.apache.flume.flume-ng-sources</groupId>
<artifactId>flume-jms-source</artifactId>
</dependency>
- <dependency>
+ <!--<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-twitter-source</artifactId>
- </dependency>
+ </dependency>-->
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-kafka-source</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-legacy-avro-source</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-legacy-thrift-source</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.flume.flume-ng-sources</groupId>
<artifactId>flume-taildir-source</artifactId>
@@ -204,7 +204,7 @@
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-config-filter-api</artifactId>
</dependency>
- <dependency>
+ <!--<dependency>
<groupId>org.apache.flume.flume-ng-clients</groupId>
<artifactId>flume-ng-log4jappender</artifactId>
<exclusions>
@@ -224,7 +224,7 @@
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
- </dependency>
+ </dependency>-->
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-tools</artifactId>
diff --git a/flume-ng-embedded-agent/pom.xml b/flume-ng-embedded-agent/pom.xml
index 79ba5595c..515321118 100644
--- a/flume-ng-embedded-agent/pom.xml
+++ b/flume-ng-embedded-agent/pom.xml
@@ -21,7 +21,7 @@ limitations under the License.
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../flume-parent/pom.xml</relativePath>
</parent>
@@ -62,6 +62,12 @@ limitations under the License.
<artifactId>flume-file-channel</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-rpc-avro</artifactId>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
diff --git a/flume-ng-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedAgent.java b/flume-ng-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedAgent.java
index a3f2c8d01..45c9e43e2 100644
--- a/flume-ng-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedAgent.java
+++ b/flume-ng-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedAgent.java
@@ -23,7 +23,7 @@ import java.util.Map;
import java.util.TreeSet;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.FlumeException;
diff --git a/flume-ng-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedSource.java b/flume-ng-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedSource.java
index 71a88ec25..375e27bcd 100644
--- a/flume-ng-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedSource.java
+++ b/flume-ng-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedSource.java
@@ -20,7 +20,7 @@ package org.apache.flume.agent.embedded;
import java.util.List;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDrivenSource;
diff --git a/flume-ng-node/pom.xml b/flume-ng-node/pom.xml
index 2f7453f66..f4c4b4c70 100644
--- a/flume-ng-node/pom.xml
+++ b/flume-ng-node/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../flume-parent/pom.xml</relativePath>
</parent>
@@ -60,14 +60,14 @@
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-hdfs-sink</artifactId>
- </dependency>-->
+ </dependency>
<dependency>
<groupId>org.apache.flume.flume-ng-sinks</groupId>
<artifactId>flume-irc-sink</artifactId>
</dependency>
- <!--<dependency>
+ <dependency>
<groupId>org.apache.flume.flume-ng-channels</groupId>
<artifactId>flume-jdbc-channel</artifactId>
</dependency>-->
diff --git a/flume-ng-node/src/test/java/org/apache/flume/node/TestAbstractConfigurationProvider.java b/flume-ng-node/src/test/java/org/apache/flume/node/TestAbstractConfigurationProvider.java
index 7810c5bca..07881b254 100644
--- a/flume-ng-node/src/test/java/org/apache/flume/node/TestAbstractConfigurationProvider.java
+++ b/flume-ng-node/src/test/java/org/apache/flume/node/TestAbstractConfigurationProvider.java
@@ -20,7 +20,7 @@ package org.apache.flume.node;
import com.google.common.collect.Maps;
import junit.framework.Assert;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
@@ -185,56 +185,6 @@ public class TestAbstractConfigurationProvider {
Assert.assertTrue(config.getSinkRunners().size() == 0);
}
- @Test
- public void testSinkSourceMismatchDuringConfiguration() throws Exception {
- String agentName = "agent1";
- String sourceType = "seq";
- String channelType = "memory";
- String sinkType = "avro";
- Map<String, String> properties = getProperties(agentName, sourceType,
- channelType, sinkType);
- properties.put(agentName + ".channels.channel1.capacity", "1000");
- properties.put(agentName + ".channels.channel1.transactionCapacity", "1000");
- properties.put(agentName + ".sources.source1.batchSize", "1000");
- properties.put(agentName + ".sinks.sink1.batch-size", "1000");
- properties.put(agentName + ".sinks.sink1.hostname", "10.10.10.10");
- properties.put(agentName + ".sinks.sink1.port", "1010");
-
- MemoryConfigurationProvider provider =
- new MemoryConfigurationProvider(agentName, properties);
- MaterializedConfiguration config = provider.getConfiguration();
- Assert.assertTrue(config.getSourceRunners().size() == 1);
- Assert.assertTrue(config.getChannels().size() == 1);
- Assert.assertTrue(config.getSinkRunners().size() == 1);
-
- properties.put(agentName + ".sources.source1.batchSize", "1001");
- properties.put(agentName + ".sinks.sink1.batch-size", "1000");
-
- provider = new MemoryConfigurationProvider(agentName, properties);
- config = provider.getConfiguration();
- Assert.assertTrue(config.getSourceRunners().size() == 0);
- Assert.assertTrue(config.getChannels().size() == 1);
- Assert.assertTrue(config.getSinkRunners().size() == 1);
-
- properties.put(agentName + ".sources.source1.batchSize", "1000");
- properties.put(agentName + ".sinks.sink1.batch-size", "1001");
-
- provider = new MemoryConfigurationProvider(agentName, properties);
- config = provider.getConfiguration();
- Assert.assertTrue(config.getSourceRunners().size() == 1);
- Assert.assertTrue(config.getChannels().size() == 1);
- Assert.assertTrue(config.getSinkRunners().size() == 0);
-
- properties.put(agentName + ".sources.source1.batchSize", "1001");
- properties.put(agentName + ".sinks.sink1.batch-size", "1001");
-
- provider = new MemoryConfigurationProvider(agentName, properties);
- config = provider.getConfiguration();
- Assert.assertTrue(config.getSourceRunners().size() == 0);
- Assert.assertTrue(config.getChannels().size() == 0);
- Assert.assertTrue(config.getSinkRunners().size() == 0);
- }
-
private Map<String, String> getProperties(String agentName,
String sourceType, String channelType,
String sinkType) {
diff --git a/flume-ng-sdk/pom.xml b/flume-ng-sdk/pom.xml
index 08d0aa32e..185004bd4 100644
--- a/flume-ng-sdk/pom.xml
+++ b/flume-ng-sdk/pom.xml
@@ -21,7 +21,7 @@ limitations under the License.
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../flume-parent/pom.xml</relativePath>
</parent>
@@ -36,149 +36,8 @@ limitations under the License.
<module.name>org.apache.flume.sdk</module.name>
</properties>
- <profiles>
- <profile>
- <id>compileThrift</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>compileThrift</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <echo file="target/compile-thrift.sh">
- LICENSE=src/main/thrift/aslv2
- THRIFT_DIR=src/main/thrift
- JAVA_DIR=target/generated-sources/thrift
- mkdir -p $JAVA_DIR 2> /dev/null
- JSTATUS=$?
- if [ $JSTATUS -ne 0 ] ; then
- echo "Could not create $JAVA_DIR. Will not generate thrift files."
- exit $JSTATUS
- fi
- for THRIFT_FILE in `ls $THRIFT_DIR/*.thrift 2> /dev/null`
- do
- thrift --gen java:hashcode -o $JAVA_DIR $THRIFT_FILE
- done
- SRC_DIR=$JAVA_DIR/gen-java/org/apache/flume/thrift/
- DEST_DIR=src/main/java/org/apache/flume/thrift
- if [ ! -d $DEST_DIR ] ; then
- mkdir $DEST_DIR 2> /dev/null
- STATUS=$?
- if [ $STATUS -ne 0 ] ; then
- echo "Could not create $DEST_DIR. Will not generate thrift files."
- exit $STATUS
- fi
- fi
- for JAVA_FILE in `ls $SRC_DIR/*.java 2> /dev/null`
- do
- echo $JAVA_FILE
- cat $LICENSE > $JAVA_FILE.tmp
- cat $JAVA_FILE >> $JAVA_FILE.tmp
- mv $JAVA_FILE.tmp $JAVA_FILE
- cp $JAVA_FILE $DEST_DIR
- done
- rm -rf $JAVA_DIR
- </echo>
- <exec executable="sh" dir="${basedir}" failonerror="true">
- <arg line="target/compile-thrift.sh"/>
- </exec>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
<build>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.eclipse.m2e</groupId>
- <artifactId>lifecycle-mapping</artifactId>
- <version>${lifecycle-mapping.version}</version>
- <configuration>
- <lifecycleMappingMetadata>
- <pluginExecutions>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-maven-plugin</artifactId>
- <versionRange>[${avro.version},)</versionRange>
- <goals>
- <goal>idl-protocol</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <execute />
- </action>
- </pluginExecution>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer-maven-plugin</artifactId>
- <versionRange>[${mvn-paranamer-plugin.version},)</versionRange>
- <goals>
- <goal>generate</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <ignore />
- </action>
- </pluginExecution>
- </pluginExecutions>
- </lifecycleMappingMetadata>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
<plugins>
-
- <plugin>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-maven-plugin</artifactId>
- <configuration>
- <stringType>CharSequence</stringType>
- </configuration>
- <executions>
- <execution>
- <phase>generate-sources</phase>
- <goals>
- <goal>idl-protocol</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>run</id>
- <configuration>
- <sourceDirectory>${project.build.directory}/generated-sources/avro</sourceDirectory>
- <outputDirectory>${project.build.directory}/classes</outputDirectory>
- </configuration>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
@@ -206,8 +65,11 @@ limitations under the License.
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
- <!-- only used for tests -->
- <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
</dependency>
<dependency>
@@ -228,31 +90,6 @@ limitations under the License.
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-ipc-netty</artifactId>
- </dependency>
-
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.jcraft</groupId>
- <artifactId>jzlib</artifactId>
- </dependency>
-
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
diff --git a/flume-ng-configuration/src/main/java/org/apache/flume/Context.java b/flume-ng-sdk/src/main/java/org/apache/flume/Context.java
similarity index 100%
rename from flume-ng-configuration/src/main/java/org/apache/flume/Context.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/Context.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/annotations/Disposable.java b/flume-ng-sdk/src/main/java/org/apache/flume/annotations/Disposable.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/annotations/Disposable.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/annotations/Disposable.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/annotations/InterfaceAudience.java b/flume-ng-sdk/src/main/java/org/apache/flume/annotations/InterfaceAudience.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/annotations/InterfaceAudience.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/annotations/InterfaceAudience.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/annotations/InterfaceStability.java b/flume-ng-sdk/src/main/java/org/apache/flume/annotations/InterfaceStability.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/annotations/InterfaceStability.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/annotations/InterfaceStability.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/annotations/Recyclable.java b/flume-ng-sdk/src/main/java/org/apache/flume/annotations/Recyclable.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/annotations/Recyclable.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/annotations/Recyclable.java
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/api/FailoverRpcClient.java b/flume-ng-sdk/src/main/java/org/apache/flume/api/FailoverRpcClient.java
index b6d89d366..66d7bd96f 100644
--- a/flume-ng-sdk/src/main/java/org/apache/flume/api/FailoverRpcClient.java
+++ b/flume-ng-sdk/src/main/java/org/apache/flume/api/FailoverRpcClient.java
@@ -224,7 +224,7 @@ public class FailoverRpcClient extends AbstractRpcClient implements RpcClient {
* to another host. To be used only from the unit tests!
* @return The last socket address this client connected to
*/
- protected InetSocketAddress getLastConnectedServerAddress() {
+ public InetSocketAddress getLastConnectedServerAddress() {
HostInfo hostInfo = hosts.get(lastCheckedhost);
return new InetSocketAddress(hostInfo.getHostName(),
hostInfo.getPortNumber());
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/api/RpcClientFactory.java b/flume-ng-sdk/src/main/java/org/apache/flume/api/RpcClientFactory.java
index 4326b579a..b8ea8f61b 100644
--- a/flume-ng-sdk/src/main/java/org/apache/flume/api/RpcClientFactory.java
+++ b/flume-ng-sdk/src/main/java/org/apache/flume/api/RpcClientFactory.java
@@ -25,6 +25,7 @@ import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
+import java.lang.reflect.InvocationTargetException;
import java.util.Locale;
import java.util.Properties;
@@ -33,13 +34,17 @@ import java.util.Properties;
*/
public class RpcClientFactory {
+ private static final String AVRO_CLASS_NAME = "org.apache.flume.rpc.avro.client.NettyAvroRpcClient";
+ private static final String THRIFT_CLASS_NAME = "org.apache.flume.rpc.thrift.client.ThriftRpcClient";
+ private static final String NEW_INSTANCE = "newInstance";
+
/**
* Returns an instance of {@link RpcClient}, optionally with failover.
* To create a failover client, the properties object should have a
* property <tt>client.type</tt> which has the value "failover". The client
* connects to hosts specified by <tt>hosts</tt> property in given properties.
*
- * @see org.apache.flume.api.FailoverRpcClient
+ * @see FailoverRpcClient
* <p>
* If no <tt>client.type</tt> is specified, a default client that connects to
* single host at a given port is created.(<tt>type</tt> can also simply be
@@ -57,18 +62,22 @@ public class RpcClientFactory {
type = properties.getProperty(
RpcClientConfigurationConstants.CONFIG_CLIENT_TYPE);
if (type == null || type.isEmpty()) {
- type = ClientType.DEFAULT.getClientClassName();
+ type = ClientType.DEFAULT.name();
+ }
+ ClientType clientType = null;
+ try {
+ clientType = ClientType.valueOf(type.toUpperCase(Locale.ENGLISH));
+ } catch (IllegalArgumentException e) {
+ clientType = ClientType.OTHER;
}
- Class<? extends AbstractRpcClient> clazz;
AbstractRpcClient client;
+ if (clientType.isNewInstance()) {
+ return createClient(clientType.clientClassName, properties);
+ }
+ Class<? extends AbstractRpcClient> clazz;
try {
String clientClassType = type;
- ClientType clientType = null;
- try {
- clientType = ClientType.valueOf(type.toUpperCase(Locale.ENGLISH));
- } catch (IllegalArgumentException e) {
- clientType = ClientType.OTHER;
- }
+
if (!clientType.equals(ClientType.OTHER)) {
clientClassType = clientType.getClientClassName();
}
@@ -81,10 +90,10 @@ public class RpcClientFactory {
try {
client = clazz.newInstance();
} catch (InstantiationException e) {
- throw new FlumeException("Cannot instantiate client. " +
- "Exception follows:", e);
+ throw new FlumeException("Cannot instantiate client " + clazz.getCanonicalName() +
+ " Exception follows:", e);
} catch (IllegalAccessException e) {
- throw new FlumeException("Cannot instantiate client. " +
+ throw new FlumeException("Cannot instantiate client " + clazz.getCanonicalName() +
"Exception follows:", e);
}
client.configure(properties);
@@ -167,9 +176,7 @@ public class RpcClientFactory {
props.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + "h1",
hostname + ":" + port.intValue());
props.setProperty(RpcClientConfigurationConstants.CONFIG_BATCH_SIZE, batchSize.toString());
- NettyAvroRpcClient client = new NettyAvroRpcClient();
- client.configure(props);
- return client;
+ return createClient(AVRO_CLASS_NAME, props);
}
/**
@@ -198,9 +205,21 @@ public class RpcClientFactory {
props.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + "h1",
hostname + ":" + port.intValue());
props.setProperty(RpcClientConfigurationConstants.CONFIG_BATCH_SIZE, batchSize.toString());
- ThriftRpcClient client = new ThriftRpcClient();
- client.configure(props);
- return client;
+ return createClient(THRIFT_CLASS_NAME, props);
+ }
+
+ private static RpcClient createClient(String className, Properties props)
+ throws FlumeException {
+ try {
+ @SuppressWarnings("unchecked")
+ Class<? extends AbstractRpcClient> clazz = (Class<? extends AbstractRpcClient>) Class.forName(className);
+ AbstractRpcClient client = (AbstractRpcClient) clazz.getMethod(NEW_INSTANCE, null).invoke(null, null);
+ client.configure(props);
+ return client;
+ } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException |
+ InvocationTargetException e) {
+ throw new FlumeException("Cannot instantiate client. Avro implementation failed to load.", e);
+ }
}
/**
@@ -233,21 +252,38 @@ public class RpcClientFactory {
public static enum ClientType {
OTHER(null),
- DEFAULT(NettyAvroRpcClient.class.getCanonicalName()),
+ DEFAULT(AVRO_CLASS_NAME, true),
DEFAULT_FAILOVER(FailoverRpcClient.class.getCanonicalName()),
DEFAULT_LOADBALANCE(LoadBalancingRpcClient.class.getCanonicalName()),
- THRIFT(ThriftRpcClient.class.getCanonicalName());
-
+ THRIFT(THRIFT_CLASS_NAME, true);
private final String clientClassName;
+ private final boolean newInstance;
private ClientType(String className) {
this.clientClassName = className;
+ this.newInstance = false;
+ }
+
+ private ClientType(String className, boolean newInstance) {
+ this.clientClassName = className;
+ this.newInstance = newInstance;
+ }
+ private static ClientType getClientType(String className) {
+ for (ClientType type : ClientType.values()) {
+ if (type.clientClassName.equals(className)) {
+ return type;
+ }
+ }
+ return OTHER;
}
protected String getClientClassName() {
return this.clientClassName;
}
+ protected boolean isNewInstance() {
+ return this.newInstance;
+ }
}
}
diff --git a/flume-ng-core/src/main/java/org/apache/flume/client/avro/EventReader.java b/flume-ng-sdk/src/main/java/org/apache/flume/event/EventReader.java
similarity index 98%
rename from flume-ng-core/src/main/java/org/apache/flume/client/avro/EventReader.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/event/EventReader.java
index 768a38185..0f00d968c 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/client/avro/EventReader.java
+++ b/flume-ng-sdk/src/main/java/org/apache/flume/event/EventReader.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.flume.client.avro;
+package org.apache.flume.event;
import org.apache.flume.Event;
import org.apache.flume.annotations.InterfaceAudience;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableEventReader.java b/flume-ng-sdk/src/main/java/org/apache/flume/event/ReliableEventReader.java
similarity index 97%
rename from flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableEventReader.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/event/ReliableEventReader.java
index 99c94c62c..c6f7a18f5 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableEventReader.java
+++ b/flume-ng-sdk/src/main/java/org/apache/flume/event/ReliableEventReader.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.flume.client.avro;
+package org.apache.flume.event;
import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/client/avro/SimpleTextLineEventReader.java b/flume-ng-sdk/src/main/java/org/apache/flume/event/SimpleTextLineEventReader.java
similarity index 96%
rename from flume-ng-core/src/main/java/org/apache/flume/client/avro/SimpleTextLineEventReader.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/event/SimpleTextLineEventReader.java
index 2fcb15252..799b311b2 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/client/avro/SimpleTextLineEventReader.java
+++ b/flume-ng-sdk/src/main/java/org/apache/flume/event/SimpleTextLineEventReader.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.flume.client.avro;
+package org.apache.flume.event;
import java.io.BufferedReader;
import java.io.IOException;
@@ -29,7 +29,6 @@ import com.google.common.collect.Lists;
import org.apache.flume.Event;
import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
-import org.apache.flume.event.EventBuilder;
/**
* A {@link EventReader} implementation which delegates to a
diff --git a/flume-ng-core/src/main/java/org/apache/flume/ChannelException.java b/flume-ng-sdk/src/main/java/org/apache/flume/exception/ChannelException.java
similarity index 97%
rename from flume-ng-core/src/main/java/org/apache/flume/ChannelException.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/exception/ChannelException.java
index 2ac0b76f0..c2e0e5488 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/ChannelException.java
+++ b/flume-ng-sdk/src/main/java/org/apache/flume/exception/ChannelException.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.flume;
+package org.apache.flume.exception;
/**
* <p>A channel exception is raised whenever a channel operation fails.</p>
diff --git a/flume-ng-core/src/main/java/org/apache/flume/ChannelFullException.java b/flume-ng-sdk/src/main/java/org/apache/flume/exception/ChannelFullException.java
similarity index 97%
rename from flume-ng-core/src/main/java/org/apache/flume/ChannelFullException.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/exception/ChannelFullException.java
index 675ef13f7..96fade44c 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/ChannelFullException.java
+++ b/flume-ng-sdk/src/main/java/org/apache/flume/exception/ChannelFullException.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.flume;
+package org.apache.flume.exception;
public class ChannelFullException extends ChannelException {
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/ChannelCounter.java b/flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/ChannelCounter.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/ChannelCounter.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/ChannelCounter.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/ChannelCounterMBean.java b/flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/ChannelCounterMBean.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/ChannelCounterMBean.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/ChannelCounterMBean.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/MonitoredCounterGroup.java b/flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/MonitoredCounterGroup.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/MonitoredCounterGroup.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/MonitoredCounterGroup.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/SinkCounter.java b/flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/SinkCounter.java
similarity index 98%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/SinkCounter.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/SinkCounter.java
index 8c77959be..77d857752 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/SinkCounter.java
+++ b/flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/SinkCounter.java
@@ -18,7 +18,7 @@
package org.apache.flume.instrumentation;
import org.apache.commons.lang.ArrayUtils;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
public class SinkCounter extends MonitoredCounterGroup implements
SinkCounterMBean {
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/SinkCounterMBean.java b/flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/SinkCounterMBean.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/SinkCounterMBean.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/SinkCounterMBean.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/SourceCounter.java b/flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/SourceCounter.java
similarity index 98%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/SourceCounter.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/SourceCounter.java
index 0e4d97100..46c1f2d98 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/SourceCounter.java
+++ b/flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/SourceCounter.java
@@ -19,7 +19,7 @@
package org.apache.flume.instrumentation;
import org.apache.commons.lang.ArrayUtils;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
public class SourceCounter extends MonitoredCounterGroup implements
SourceCounterMBean {
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/SourceCounterMBean.java b/flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/SourceCounterMBean.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/SourceCounterMBean.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/instrumentation/SourceCounterMBean.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/DecodeErrorPolicy.java b/flume-ng-sdk/src/main/java/org/apache/flume/serialization/DecodeErrorPolicy.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/DecodeErrorPolicy.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/serialization/DecodeErrorPolicy.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializer.java b/flume-ng-sdk/src/main/java/org/apache/flume/serialization/EventDeserializer.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializer.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/serialization/EventDeserializer.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/LengthMeasurable.java b/flume-ng-sdk/src/main/java/org/apache/flume/serialization/LengthMeasurable.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/LengthMeasurable.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/serialization/LengthMeasurable.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/LineDeserializer.java b/flume-ng-sdk/src/main/java/org/apache/flume/serialization/LineDeserializer.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/LineDeserializer.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/serialization/LineDeserializer.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/PositionTracker.java b/flume-ng-sdk/src/main/java/org/apache/flume/serialization/PositionTracker.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/PositionTracker.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/serialization/PositionTracker.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/RemoteMarkable.java b/flume-ng-sdk/src/main/java/org/apache/flume/serialization/RemoteMarkable.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/RemoteMarkable.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/serialization/RemoteMarkable.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/Resettable.java b/flume-ng-sdk/src/main/java/org/apache/flume/serialization/Resettable.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/Resettable.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/serialization/Resettable.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/ResettableFileInputStream.java b/flume-ng-sdk/src/main/java/org/apache/flume/serialization/ResettableFileInputStream.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/ResettableFileInputStream.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/serialization/ResettableFileInputStream.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/ResettableInputStream.java b/flume-ng-sdk/src/main/java/org/apache/flume/serialization/ResettableInputStream.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/ResettableInputStream.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/serialization/ResettableInputStream.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/Seekable.java b/flume-ng-sdk/src/main/java/org/apache/flume/serialization/Seekable.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/Seekable.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/serialization/Seekable.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySourceConfigurationConstants.java b/flume-ng-sdk/src/main/java/org/apache/flume/source/SpoolDirectorySourceConfigurationConstants.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySourceConfigurationConstants.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/source/SpoolDirectorySourceConfigurationConstants.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/tools/PlatformDetect.java b/flume-ng-sdk/src/main/java/org/apache/flume/tools/PlatformDetect.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/tools/PlatformDetect.java
rename to flume-ng-sdk/src/main/java/org/apache/flume/tools/PlatformDetect.java
diff --git a/flume-ng-sdk/src/test/java/org/apache/flume/api/ThriftTestingSource.java b/flume-ng-sdk/src/test/java/org/apache/flume/api/ThriftTestingSource.java
deleted file mode 100644
index 059343221..000000000
--- a/flume-ng-sdk/src/test/java/org/apache/flume/api/ThriftTestingSource.java
+++ /dev/null
@@ -1,282 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.api;
-
-import org.apache.flume.Event;
-import org.apache.flume.FlumeException;
-import org.apache.flume.event.EventBuilder;
-import org.apache.flume.thrift.Status;
-import org.apache.flume.thrift.ThriftFlumeEvent;
-import org.apache.flume.thrift.ThriftSourceProtocol;
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.protocol.TCompactProtocol;
-import org.apache.thrift.protocol.TProtocolFactory;
-import org.apache.thrift.server.THsHaServer;
-import org.apache.thrift.server.TServer;
-import org.apache.thrift.transport.TSSLTransportFactory;
-import org.apache.thrift.transport.TNonblockingServerSocket;
-import org.apache.thrift.transport.TServerSocket;
-import org.apache.thrift.transport.TNonblockingServerTransport;
-import org.apache.thrift.transport.TServerTransport;
-import org.apache.thrift.transport.layered.TFastFramedTransport;
-
-import java.lang.reflect.Method;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-public class ThriftTestingSource {
- public final Queue<Event> flumeEvents = new ConcurrentLinkedQueue<Event>();
- private final TServer server;
- public int batchCount = 0;
- public int individualCount = 0;
- public int incompleteBatches = 0;
- private AtomicLong delay = null;
-
- public void setDelay(AtomicLong delay) {
- this.delay = delay;
- }
-
- private class ThriftOKHandler implements ThriftSourceProtocol.Iface {
-
- public ThriftOKHandler() {
-
- }
-
- @Override
- public Status append(ThriftFlumeEvent event) throws TException {
- flumeEvents.add(EventBuilder.withBody(event.getBody(), event.getHeaders()));
- individualCount++;
- return Status.OK;
- }
-
- @Override
- public Status appendBatch(List<ThriftFlumeEvent> events) throws TException {
- batchCount++;
- if (events.size() < 10) {
- incompleteBatches++;
- }
- for (ThriftFlumeEvent event : events) {
- flumeEvents.add(EventBuilder.withBody(event.getBody(), event.getHeaders()));
- }
- return Status.OK;
- }
- }
-
- private class ThriftFailHandler implements ThriftSourceProtocol.Iface {
-
- @Override
- public Status append(ThriftFlumeEvent event) throws TException {
- return Status.FAILED;
- }
-
- @Override
- public Status appendBatch(List<ThriftFlumeEvent> events) throws
- TException {
- return Status.FAILED;
- }
- }
-
- private class ThriftErrorHandler implements ThriftSourceProtocol.Iface {
-
- @Override
- public Status append(ThriftFlumeEvent event) throws TException {
- throw new FlumeException("Forced Error");
- }
-
- @Override
- public Status appendBatch(List<ThriftFlumeEvent> events) throws TException {
- throw new FlumeException("Forced Error");
- }
- }
-
- private class ThriftSlowHandler extends ThriftOKHandler {
-
- @Override
- public Status append(ThriftFlumeEvent event) throws TException {
- try {
- TimeUnit.MILLISECONDS.sleep(1550);
- } catch (InterruptedException e) {
- throw new FlumeException("Error", e);
- }
- return super.append(event);
- }
-
- @Override
- public Status appendBatch(List<ThriftFlumeEvent> events) throws TException {
- try {
- TimeUnit.MILLISECONDS.sleep(1550);
- } catch (InterruptedException e) {
- throw new FlumeException("Error", e);
- }
- return super.appendBatch(events);
- }
- }
-
- private class ThriftTimeoutHandler extends ThriftOKHandler {
-
- @Override
- public Status append(ThriftFlumeEvent event) throws TException {
- try {
- TimeUnit.MILLISECONDS.sleep(5000);
- } catch (InterruptedException e) {
- throw new FlumeException("Error", e);
- }
- return super.append(event);
- }
-
- @Override
- public Status appendBatch(List<ThriftFlumeEvent> events) throws TException {
- try {
- TimeUnit.MILLISECONDS.sleep(5000);
- } catch (InterruptedException e) {
- throw new FlumeException("Error", e);
- }
- return super.appendBatch(events);
- }
- }
-
- private class ThriftAlternateHandler extends ThriftOKHandler {
-
- @Override
- public Status append(ThriftFlumeEvent event) throws TException {
- try {
- if (delay != null) {
- TimeUnit.MILLISECONDS.sleep(delay.get());
- }
- } catch (InterruptedException e) {
- throw new FlumeException("Error", e);
- }
- return super.append(event);
- }
-
- @Override
- public Status appendBatch(List<ThriftFlumeEvent> events) throws TException {
- try {
- if (delay != null) {
- TimeUnit.MILLISECONDS.sleep(delay.get());
- }
- } catch (InterruptedException e) {
- throw new FlumeException("Error", e);
- }
- return super.appendBatch(events);
- }
- }
-
- private ThriftSourceProtocol.Iface getHandler(String handlerName) {
- ThriftSourceProtocol.Iface handler = null;
- if (handlerName.equals(HandlerType.OK.name())) {
- handler = new ThriftOKHandler();
- } else if (handlerName.equals(HandlerType.FAIL.name())) {
- handler = new ThriftFailHandler();
- } else if (handlerName.equals(HandlerType.ERROR.name())) {
- handler = new ThriftErrorHandler();
- } else if (handlerName.equals(HandlerType.SLOW.name())) {
- handler = new ThriftSlowHandler();
- } else if (handlerName.equals(HandlerType.TIMEOUT.name())) {
- handler = new ThriftTimeoutHandler();
- } else if (handlerName.equals(HandlerType.ALTERNATE.name())) {
- handler = new ThriftAlternateHandler();
- }
- return handler;
- }
-
- public ThriftTestingSource(String handlerName, int port, String protocol) throws Exception {
- TNonblockingServerTransport serverTransport =
- new TNonblockingServerSocket(new InetSocketAddress("0.0.0.0", port));
- ThriftSourceProtocol.Iface handler = getHandler(handlerName);
-
- TProtocolFactory transportProtocolFactory = null;
- if (protocol != null && protocol == ThriftRpcClient.BINARY_PROTOCOL) {
- transportProtocolFactory = new TBinaryProtocol.Factory();
- } else {
- transportProtocolFactory = new TCompactProtocol.Factory();
- }
- server = new THsHaServer(new THsHaServer.Args(serverTransport).processor(
- new ThriftSourceProtocol.Processor(handler)).protocolFactory(
- transportProtocolFactory));
- Executors.newSingleThreadExecutor().submit(new Runnable() {
- @Override
- public void run() {
- server.serve();
- }
- });
- }
-
- public ThriftTestingSource(String handlerName, int port,
- String protocol, String keystore,
- String keystorePassword, String keyManagerType,
- String keystoreType) throws Exception {
- TSSLTransportFactory.TSSLTransportParameters params =
- new TSSLTransportFactory.TSSLTransportParameters();
- params.setKeyStore(keystore, keystorePassword, keyManagerType, keystoreType);
-
- TServerSocket serverTransport = TSSLTransportFactory.getServerSocket(
- port, 10000, InetAddress.getByName("0.0.0.0"), params);
-
- ThriftSourceProtocol.Iface handler = getHandler(handlerName);
-
- Class serverClass = Class.forName("org.apache.thrift" +
- ".server.TThreadPoolServer");
- Class argsClass = Class.forName("org.apache.thrift.server" +
- ".TThreadPoolServer$Args");
- TServer.AbstractServerArgs args = (TServer.AbstractServerArgs) argsClass
- .getConstructor(TServerTransport.class)
- .newInstance(serverTransport);
- Method m = argsClass.getDeclaredMethod("maxWorkerThreads", int.class);
- m.invoke(args, Integer.MAX_VALUE);
- TProtocolFactory transportProtocolFactory = null;
- if (protocol != null && protocol == ThriftRpcClient.BINARY_PROTOCOL) {
- transportProtocolFactory = new TBinaryProtocol.Factory();
- } else {
- transportProtocolFactory = new TCompactProtocol.Factory();
- }
- args.protocolFactory(transportProtocolFactory);
- args.inputTransportFactory(new TFastFramedTransport.Factory());
- args.outputTransportFactory(new TFastFramedTransport.Factory());
- args.processor(new ThriftSourceProtocol.Processor<ThriftSourceProtocol.Iface>(handler));
- server = (TServer) serverClass.getConstructor(argsClass).newInstance(args);
- Executors.newSingleThreadExecutor().submit(new Runnable() {
- @Override
- public void run() {
- server.serve();
- }
- });
- }
-
- public enum HandlerType {
- OK,
- FAIL,
- ERROR,
- SLOW,
- TIMEOUT,
- ALTERNATE;
- }
-
- public void stop() {
- server.stop();
- }
-
-}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestBufferedLineReader.java b/flume-ng-sdk/src/test/java/org/apache/flume/event/TestBufferedLineReader.java
similarity index 89%
rename from flume-ng-core/src/test/java/org/apache/flume/client/avro/TestBufferedLineReader.java
rename to flume-ng-sdk/src/test/java/org/apache/flume/event/TestBufferedLineReader.java
index 2c7050205..1f138bed5 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestBufferedLineReader.java
+++ b/flume-ng-sdk/src/test/java/org/apache/flume/event/TestBufferedLineReader.java
@@ -17,18 +17,19 @@
* under the License.
*/
-package org.apache.flume.client.avro;
+package org.apache.flume.event;
-import static org.apache.flume.client.avro.TestSpoolingFileLineReader.bodyAsString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import static org.apache.flume.client.avro.TestSpoolingFileLineReader.bodiesAsStrings;
+
+import com.google.common.collect.Lists;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.List;
+import org.apache.flume.Event;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -115,5 +116,16 @@ public class TestBufferedLineReader {
assertTrue(out.contains("file1line8"));
}
+ static String bodyAsString(Event event) {
+ return new String(event.getBody());
+ }
+
+ static List<String> bodiesAsStrings(List<Event> events) {
+ List<String> bodies = Lists.newArrayListWithCapacity(events.size());
+ for (Event event : events) {
+ bodies.add(bodyAsString(event));
+ }
+ return bodies;
+ }
}
diff --git a/flume-ng-sinks/flume-http-sink/pom.xml b/flume-ng-sinks/flume-http-sink/pom.xml
index 936cfda07..d4ac3f52a 100644
--- a/flume-ng-sinks/flume-http-sink/pom.xml
+++ b/flume-ng-sinks/flume-http-sink/pom.xml
@@ -23,7 +23,7 @@ limitations under the License.
<parent>
<artifactId>flume-ng-sinks</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.flume.flume-ng-sinks</groupId>
diff --git a/flume-ng-sinks/flume-irc-sink/pom.xml b/flume-ng-sinks/flume-irc-sink/pom.xml
index c40eaddb8..53a5f4fe6 100644
--- a/flume-ng-sinks/flume-irc-sink/pom.xml
+++ b/flume-ng-sinks/flume-irc-sink/pom.xml
@@ -23,7 +23,7 @@ limitations under the License.
<parent>
<artifactId>flume-ng-sinks</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.flume.flume-ng-sinks</groupId>
diff --git a/flume-ng-sinks/flume-irc-sink/src/main/java/org/apache/flume/sink/irc/IRCSink.java b/flume-ng-sinks/flume-irc-sink/src/main/java/org/apache/flume/sink/irc/IRCSink.java
index 52bbfc853..5e7400c60 100644
--- a/flume-ng-sinks/flume-irc-sink/src/main/java/org/apache/flume/sink/irc/IRCSink.java
+++ b/flume-ng-sinks/flume-irc-sink/src/main/java/org/apache/flume/sink/irc/IRCSink.java
@@ -20,13 +20,13 @@ package org.apache.flume.sink.irc;
import java.io.IOException;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.CounterGroup;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.Transaction;
import org.apache.flume.conf.Configurable;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.sink.AbstractSink;
import org.schwering.irc.lib.IRCConnection;
import org.schwering.irc.lib.IRCEventListener;
diff --git a/flume-ng-sinks/pom.xml b/flume-ng-sinks/pom.xml
index 806cb1afa..5f6240a9c 100644
--- a/flume-ng-sinks/pom.xml
+++ b/flume-ng-sinks/pom.xml
@@ -23,7 +23,7 @@ limitations under the License.
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../flume-parent/pom.xml</relativePath>
</parent>
diff --git a/flume-ng-sources/flume-jms-source/pom.xml b/flume-ng-sources/flume-jms-source/pom.xml
index 7c2bd61ec..f34b767b0 100644
--- a/flume-ng-sources/flume-jms-source/pom.xml
+++ b/flume-ng-sources/flume-jms-source/pom.xml
@@ -23,7 +23,7 @@ limitations under the License.
<parent>
<artifactId>flume-ng-sources</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.flume.flume-ng-sources</groupId>
diff --git a/flume-ng-sources/flume-jms-source/src/main/java/org/apache/flume/source/jms/JMSSource.java b/flume-ng-sources/flume-jms-source/src/main/java/org/apache/flume/source/jms/JMSSource.java
index 43df14518..e6f8a11d1 100644
--- a/flume-ng-sources/flume-jms-source/src/main/java/org/apache/flume/source/jms/JMSSource.java
+++ b/flume-ng-sources/flume-jms-source/src/main/java/org/apache/flume/source/jms/JMSSource.java
@@ -33,7 +33,7 @@ import javax.naming.InitialContext;
import javax.naming.NamingException;
import org.apache.commons.lang.StringUtils;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
diff --git a/flume-ng-sources/flume-jms-source/src/test/java/org/apache/flume/source/jms/TestJMSSource.java b/flume-ng-sources/flume-jms-source/src/test/java/org/apache/flume/source/jms/TestJMSSource.java
index e4c8cb442..ea17284f2 100644
--- a/flume-ng-sources/flume-jms-source/src/test/java/org/apache/flume/source/jms/TestJMSSource.java
+++ b/flume-ng-sources/flume-jms-source/src/test/java/org/apache/flume/source/jms/TestJMSSource.java
@@ -33,7 +33,7 @@ import javax.naming.NamingException;
import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
diff --git a/flume-ng-sources/flume-scribe-source/pom.xml b/flume-ng-sources/flume-scribe-source/pom.xml
index dd5e082e0..83405dc65 100644
--- a/flume-ng-sources/flume-scribe-source/pom.xml
+++ b/flume-ng-sources/flume-scribe-source/pom.xml
@@ -23,7 +23,7 @@ limitations under the License.
<parent>
<artifactId>flume-ng-sources</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.flume.flume-ng-sources</groupId>
diff --git a/flume-ng-sources/flume-taildir-source/pom.xml b/flume-ng-sources/flume-taildir-source/pom.xml
index b5a74d7c4..8217aafd0 100644
--- a/flume-ng-sources/flume-taildir-source/pom.xml
+++ b/flume-ng-sources/flume-taildir-source/pom.xml
@@ -23,7 +23,7 @@ limitations under the License.
<parent>
<artifactId>flume-ng-sources</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.flume.flume-ng-sources</groupId>
diff --git a/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/ReliableTaildirEventReader.java b/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/ReliableTaildirEventReader.java
index ae9583620..c8ad74d2b 100644
--- a/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/ReliableTaildirEventReader.java
+++ b/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/ReliableTaildirEventReader.java
@@ -29,7 +29,7 @@ import org.apache.flume.Event;
import org.apache.flume.FlumeException;
import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
-import org.apache.flume.client.avro.ReliableEventReader;
+import org.apache.flume.event.ReliableEventReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java b/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java
index 9ecccd748..73c6911e5 100644
--- a/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java
+++ b/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java
@@ -34,7 +34,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.FlumeException;
diff --git a/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirSource.java b/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirSource.java
index 1c30cd4a0..a236f30c4 100644
--- a/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirSource.java
+++ b/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirSource.java
@@ -25,7 +25,7 @@ import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.ChannelSelector;
import org.apache.flume.Context;
import org.apache.flume.Event;
diff --git a/flume-ng-sources/pom.xml b/flume-ng-sources/pom.xml
index 4ecf06f4b..d0dcbc4f3 100644
--- a/flume-ng-sources/pom.xml
+++ b/flume-ng-sources/pom.xml
@@ -23,7 +23,7 @@ limitations under the License.
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../flume-parent/pom.xml</relativePath>
</parent>
@@ -33,7 +33,6 @@ limitations under the License.
<packaging>pom</packaging>
<modules>
- <module>flume-scribe-source</module>
<module>flume-jms-source</module>
<module>flume-taildir-source</module>
</modules>
diff --git a/flume-ng-tests/pom.xml b/flume-ng-tests/pom.xml
index da3ecfa43..20ec4cc94 100644
--- a/flume-ng-tests/pom.xml
+++ b/flume-ng-tests/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../flume-parent/pom.xml</relativePath>
</parent>
@@ -45,6 +45,10 @@
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-configuration</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-rpc-avro</artifactId>
+ </dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
diff --git a/flume-ng-tests/src/test/resources/log4j2.xml b/flume-ng-tests/src/test/resources/log4j2.xml
index a6e2e30bd..517069a2d 100644
--- a/flume-ng-tests/src/test/resources/log4j2.xml
+++ b/flume-ng-tests/src/test/resources/log4j2.xml
@@ -23,7 +23,7 @@
</Console>
</Appenders>
<Loggers>
- <Root level="INFO">
+ <Root level="TRACE">
<Logger name="org.apache.flume" level="DEBUG"/>
<AppenderRef ref="Console" />
</Root>
diff --git a/flume-parent/pom.xml b/flume-parent/pom.xml
index 4e4203e76..5997b66c4 100644
--- a/flume-parent/pom.xml
+++ b/flume-parent/pom.xml
@@ -27,7 +27,7 @@ limitations under the License.
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.flume</groupId>
<artifactId>flume-parent</artifactId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Flume</name>
@@ -52,7 +52,6 @@ limitations under the License.
<activemq.version>5.7.0</activemq.version>
<asynchbase.version>1.8.2</asynchbase.version>
- <avro.version>1.11.1</avro.version>
<bundle-plugin.version>2.3.7</bundle-plugin.version>
<checkstyle.tool.version>8.45.1</checkstyle.tool.version>
<commons-cli.version>1.5.0</commons-cli.version>
@@ -122,27 +121,17 @@ limitations under the License.
<solr-global.version>4.3.0</solr-global.version>
<slf4j.version>1.7.32</slf4j.version>
<system-rules.version>1.17.0</system-rules.version>
- <thrift.version>0.14.2</thrift.version>
<twitter-source.version>2.0.0</twitter-source.version>
<wiremock.version>1.53</wiremock.version>
<zlib.version>1.1.3</zlib.version>
<zookeeper.version>3.6.2</zookeeper.version>
<os.maven.version>1.5.0.Final</os.maven.version>
<redirectTestOutput>true</redirectTestOutput>
+ <checkstyle.skip>true</checkstyle.skip>
</properties>
<profiles>
- <profile>
- <id>compileThriftLegacy</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <properties>
- <thrift.executable>${env.THRIFT_HOME}/bin/thrift</thrift.executable>
- </properties>
- </profile>
-
<profile>
<id>not-windows</id>
<activation>
@@ -432,17 +421,6 @@ limitations under the License.
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <dependencies>
- <dependency>
- <groupId>com.puppycrawl.tools</groupId>
- <artifactId>checkstyle</artifactId>
- <version>${checkstyle.tool.version}</version>
- </dependency>
- </dependencies>
- </plugin>
<plugin>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-maven-plugin</artifactId>
@@ -584,44 +562,6 @@ limitations under the License.
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <version>${mvn-checkstyle-plugin.version}</version>
- <dependencies>
- <!-- The build-support module adds the checkstyle config files to the classpath. -->
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>build-support</artifactId>
- <version>${project.version}</version>
- </dependency>
- <!-- Newer versions of puppycrawl checkstyle have more features. Pull in a more recent
- version than is specified in the maven-checkstyle-plugin pom file. -->
- <dependency>
- <groupId>com.puppycrawl.tools</groupId>
- <artifactId>checkstyle</artifactId>
- <version>${checkstyle.tool.version}</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>verify.checkstyle</id>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <configLocation>config/checkstyle/checkstyle.xml</configLocation>
- <suppressionsLocation>config/checkstyle/checkstyle-suppressions.xml</suppressionsLocation>
- <suppressionsFileExpression>checkstyle.suppressions.file</suppressionsFileExpression>
- <encoding>UTF-8</encoding>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- <linkXRef>false</linkXRef>
- </configuration>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
@@ -645,18 +585,6 @@ limitations under the License.
<version>${mvn-assembly-plugin.version}</version>
</plugin>
- <plugin>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-maven-plugin</artifactId>
- <version>${avro.version}</version>
- </plugin>
-
- <plugin>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer-maven-plugin</artifactId>
- <version>${mvn-paranamer-plugin.version}</version>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-site-plugin</artifactId>
@@ -761,6 +689,12 @@ limitations under the License.
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-rpc-avro</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
@@ -859,54 +793,6 @@ limitations under the License.
<version>${slf4j.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- <version>${avro.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-compiler</artifactId>
- <version>${avro.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-ipc-netty</artifactId>
- <version>${avro.version}</version>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-ipc-jetty</artifactId>
- <version>${avro.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- <version>${thrift.version}</version>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
@@ -1509,30 +1395,6 @@ limitations under the License.
<version>${dropwizard-metrics.version}</version>
</dependency>
- <!-- Required for the morphlines-solr-sink via the kite-sdk -->
- <dependency>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-hive-bundle</artifactId>
- <version>${parquet.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-avro</artifactId>
- <version>${parquet.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<dependency>
<groupId>net.jcip</groupId>
<artifactId>jcip-annotations</artifactId>
diff --git a/flume-rpc/flume-rpc-avro/pom.xml b/flume-rpc/flume-rpc-avro/pom.xml
new file mode 100644
index 000000000..0b6cd6d99
--- /dev/null
+++ b/flume-rpc/flume-rpc-avro/pom.xml
@@ -0,0 +1,222 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-rpc</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-rpc-avro</artifactId>
+ <name>Flume RPC Avro</name>
+
+ <properties>
+ <module.name>org.apache.flume.rpc.avro</module.name>
+ <avro.version>1.11.1</avro.version>
+ <spotbugs.maxAllowedViolations>23</spotbugs.maxAllowedViolations>
+ <pmd.maxAllowedViolations>121</pmd.maxAllowedViolations>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-ipc-netty</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-sdk</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-configuration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-node</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${dropwizard-metrics.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-jcl</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${rat.version}</version>
+ <configuration>
+ <excludes>
+ <exclude>**/.idea/</exclude>
+ <exclude>**/*.iml</exclude>
+ <exclude>src/main/resources/META-INF/services/**/*</exclude>
+ <exclude>**/nb-configuration.xml</exclude>
+ <exclude>.git/</exclude>
+ <exclude>patchprocess/</exclude>
+ <exclude>.gitignore</exclude>
+ <exclude>**/*.yml</exclude>
+ <exclude>**/*.yaml</exclude>
+ <exclude>**/*.json</exclude>
+ <!-- ASF jenkins box puts the Maven repo in our root directory. -->
+ <exclude>.repository/</exclude>
+ <exclude>**/*.diff</exclude>
+ <exclude>**/*.patch</exclude>
+ <exclude>**/*.avsc</exclude>
+ <exclude>**/*.avro</exclude>
+ <exclude>**/docs/**</exclude>
+ <exclude>**/test/resources/**</exclude>
+ <exclude>**/.settings/*</exclude>
+ <exclude>**/.classpath</exclude>
+ <exclude>**/.project</exclude>
+ <exclude>**/target/**</exclude>
+ <exclude>**/derby.log</exclude>
+ <exclude>**/metastore_db/</exclude>
+ <exclude>.mvn/**</exclude>
+ <exclude>**/exclude-pmd.properties</exclude>
+ </excludes>
+ <consoleOutput>true</consoleOutput>
+ </configuration>
+ <executions>
+ <execution>
+ <id>verify.rat</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-maven-plugin</artifactId>
+ <configuration>
+ <stringType>CharSequence</stringType>
+ </configuration>
+ <executions>
+ <execution>
+ <id>Avro Generate</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>idl-protocol</goal>
+ <goal>schema</goal>
+ </goals>
+ <configuration>
+ <sourceDirectory>${project.basedir}/src/main/avro</sourceDirectory>
+ <outputDirectory>${project.build.directory}/generated-sources/avro</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.thoughtworks.paranamer</groupId>
+ <artifactId>paranamer-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>run</id>
+ <configuration>
+ <sourceDirectory>${project.build.directory}/generated-sources/avro</sourceDirectory>
+ <outputDirectory>${project.build.directory}/classes</outputDirectory>
+ </configuration>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/flume-ng-core/src/main/avro/TransferStateFileMeta.avsc b/flume-rpc/flume-rpc-avro/src/main/avro/TransferStateFileMeta.avsc
similarity index 100%
rename from flume-ng-core/src/main/avro/TransferStateFileMeta.avsc
rename to flume-rpc/flume-rpc-avro/src/main/avro/TransferStateFileMeta.avsc
diff --git a/flume-ng-sdk/src/main/avro/flume.avdl b/flume-rpc/flume-rpc-avro/src/main/avro/flume.avdl
similarity index 100%
rename from flume-ng-sdk/src/main/avro/flume.avdl
rename to flume-rpc/flume-rpc-avro/src/main/avro/flume.avdl
diff --git a/flume-ng-core/src/main/java/org/apache/flume/client/avro/AvroCLIClient.java b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/client/AvroCLIClient.java
similarity index 96%
rename from flume-ng-core/src/main/java/org/apache/flume/client/avro/AvroCLIClient.java
rename to flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/client/AvroCLIClient.java
index 0f0fda604..35288f777 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/client/avro/AvroCLIClient.java
+++ b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/client/AvroCLIClient.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.flume.client.avro;
+package org.apache.flume.rpc.avro.client;
import java.io.File;
import java.io.FileInputStream;
@@ -43,7 +43,11 @@ import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
import org.apache.flume.api.RpcClient;
import org.apache.flume.api.RpcClientFactory;
+import org.apache.flume.event.EventReader;
+import org.apache.flume.event.ReliableEventReader;
+import org.apache.flume.event.SimpleTextLineEventReader;
import org.apache.flume.instrumentation.SourceCounter;
+import org.apache.flume.rpc.avro.serialization.ReliableSpoolingFileEventReader;
import org.apache.flume.util.SSLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/api/NettyAvroRpcClient.java b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/client/NettyAvroRpcClient.java
similarity index 97%
rename from flume-ng-sdk/src/main/java/org/apache/flume/api/NettyAvroRpcClient.java
rename to flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/client/NettyAvroRpcClient.java
index cc4956958..9c2109315 100644
--- a/flume-ng-sdk/src/main/java/org/apache/flume/api/NettyAvroRpcClient.java
+++ b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/client/NettyAvroRpcClient.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.api;
+package org.apache.flume.rpc.avro.client;
import java.io.FileInputStream;
import java.io.IOException;
@@ -60,6 +60,9 @@ import org.apache.commons.lang.StringUtils;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.FlumeException;
+import org.apache.flume.api.AbstractRpcClient;
+import org.apache.flume.api.RpcClientConfigurationConstants;
+import org.apache.flume.api.SSLContextAwareAbstractRpcClient;
import org.apache.flume.source.avro.AvroFlumeEvent;
import org.apache.flume.source.avro.AvroSourceProtocol;
import org.apache.flume.source.avro.Status;
@@ -73,7 +76,7 @@ import io.netty.handler.codec.compression.ZlibEncoder;
import io.netty.handler.ssl.SslHandler;
/**
- * Avro/Netty implementation of {@link RpcClient}.
+ * Avro/Netty implementation of {@link org.apache.flume.rpc.api.RpcClient}.
* The connections are intended to be opened before clients are given access so
* that the object cannot ever be in an inconsistent when exposed to users.
*/
@@ -95,8 +98,12 @@ public class NettyAvroRpcClient extends SSLContextAwareAbstractRpcClient {
private boolean enableDeflateCompression;
private int compressionLevel;
+ public static NettyAvroRpcClient newInstance() {
+ return new NettyAvroRpcClient();
+ }
+
/**
- * This constructor is intended to be called from {@link RpcClientFactory}.
+ * This constructor is intended to be called from {@link org.apache.flume.rpc.api.RpcClientFactory}.
* A call to this constructor should be followed by call to configure().
*/
protected NettyAvroRpcClient() {
@@ -444,7 +451,7 @@ public class NettyAvroRpcClient extends SSLContextAwareAbstractRpcClient {
stateLock.unlock();
}
- batchSize = parseBatchSize(properties);
+ batchSize = AbstractRpcClient.parseBatchSize(properties);
// host and port
String hostNames = properties.getProperty(RpcClientConfigurationConstants.CONFIG_HOSTS);
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/AbstractAvroEventSerializer.java b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AbstractAvroEventSerializer.java
similarity index 86%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/AbstractAvroEventSerializer.java
rename to flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AbstractAvroEventSerializer.java
index 7ae77a57e..5becc2d18 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/serialization/AbstractAvroEventSerializer.java
+++ b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AbstractAvroEventSerializer.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.serialization;
+package org.apache.flume.rpc.avro.serialization;
import java.io.IOException;
import java.io.OutputStream;
@@ -31,10 +31,14 @@ import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.conf.Configurable;
+import org.apache.flume.serialization.EventSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.*;
+import static org.apache.flume.rpc.avro.serialization.AvroEventSerializerConfigurationConstants.COMPRESSION_CODEC;
+import static org.apache.flume.rpc.avro.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_COMPRESSION_CODEC;
+import static org.apache.flume.rpc.avro.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_SYNC_INTERVAL_BYTES;
+import static org.apache.flume.rpc.avro.serialization.AvroEventSerializerConfigurationConstants.SYNC_INTERVAL_BYTES;
/**
* This is a helper class provided to make it straightforward to serialize
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/AvroEventDeserializer.java b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AvroEventDeserializer.java
similarity index 96%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/AvroEventDeserializer.java
rename to flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AvroEventDeserializer.java
index e09c6bba2..dce1e49ec 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/serialization/AvroEventDeserializer.java
+++ b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AvroEventDeserializer.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.flume.serialization;
+package org.apache.flume.rpc.avro.serialization;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
@@ -34,6 +34,10 @@ import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.FlumeException;
import org.apache.flume.event.EventBuilder;
+import org.apache.flume.serialization.EventDeserializer;
+import org.apache.flume.serialization.LengthMeasurable;
+import org.apache.flume.serialization.RemoteMarkable;
+import org.apache.flume.serialization.ResettableInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/AvroEventSerializerConfigurationConstants.java b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AvroEventSerializerConfigurationConstants.java
similarity index 96%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/AvroEventSerializerConfigurationConstants.java
rename to flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AvroEventSerializerConfigurationConstants.java
index 470fcea7b..5b8007a0f 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/serialization/AvroEventSerializerConfigurationConstants.java
+++ b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AvroEventSerializerConfigurationConstants.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.serialization;
+package org.apache.flume.rpc.avro.serialization;
import org.apache.avro.file.CodecFactory;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/DurablePositionTracker.java b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/DurablePositionTracker.java
similarity index 97%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/DurablePositionTracker.java
rename to flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/DurablePositionTracker.java
index 451497d7c..16ee38883 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/serialization/DurablePositionTracker.java
+++ b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/DurablePositionTracker.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.flume.serialization;
+package org.apache.flume.rpc.avro.serialization;
import java.io.File;
import java.io.IOException;
@@ -30,6 +30,8 @@ import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
+import org.apache.flume.serialization.PositionTracker;
+import org.apache.flume.serialization.TransferStateFileMeta;
import org.apache.flume.tools.PlatformDetect;
/**
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/FlumeEventAvroEventSerializer.java b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/FlumeEventAvroEventSerializer.java
similarity index 95%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/FlumeEventAvroEventSerializer.java
rename to flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/FlumeEventAvroEventSerializer.java
index 88aab597a..599ee30ae 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/serialization/FlumeEventAvroEventSerializer.java
+++ b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/FlumeEventAvroEventSerializer.java
@@ -16,12 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.serialization;
+package org.apache.flume.rpc.avro.serialization;
import java.io.OutputStream;
import org.apache.avro.Schema;
import org.apache.flume.Context;
import org.apache.flume.Event;
+import org.apache.flume.serialization.EventSerializer;
public class FlumeEventAvroEventSerializer extends AbstractAvroEventSerializer<Event> {
diff --git a/flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/ReliableSpoolingFileEventReader.java
similarity index 99%
rename from flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java
rename to flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/ReliableSpoolingFileEventReader.java
index 41cfe961a..26e0a4660 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java
+++ b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/ReliableSpoolingFileEventReader.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.flume.client.avro;
+package org.apache.flume.rpc.avro.serialization;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
@@ -29,9 +29,9 @@ import org.apache.flume.FlumeException;
import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
import org.apache.flume.event.EventBuilder;
+import org.apache.flume.event.ReliableEventReader;
import org.apache.flume.instrumentation.SourceCounter;
import org.apache.flume.serialization.DecodeErrorPolicy;
-import org.apache.flume.serialization.DurablePositionTracker;
import org.apache.flume.serialization.EventDeserializer;
import org.apache.flume.serialization.EventDeserializerFactory;
import org.apache.flume.serialization.PositionTracker;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/sink/AvroSink.java b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/sink/AvroSink.java
similarity index 96%
rename from flume-ng-core/src/main/java/org/apache/flume/sink/AvroSink.java
rename to flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/sink/AvroSink.java
index 06991faf0..4b9d6c2b8 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/sink/AvroSink.java
+++ b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/sink/AvroSink.java
@@ -17,18 +17,19 @@
* under the License.
*/
-package org.apache.flume.sink;
-
-import java.util.Properties;
+package org.apache.flume.rpc.avro.sink;
import org.apache.flume.Channel;
import org.apache.flume.Sink;
import org.apache.flume.api.RpcClient;
import org.apache.flume.api.RpcClientFactory;
-import org.apache.flume.source.AvroSource;
+import org.apache.flume.rpc.avro.source.AvroSource;
+import org.apache.flume.sink.AbstractRpcSink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Properties;
+
/**
* <p>
* A {@link Sink} implementation that can send events to an RPC server (such as
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/AvroSource.java b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/source/AvroSource.java
similarity index 98%
rename from flume-ng-core/src/main/java/org/apache/flume/source/AvroSource.java
rename to flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/source/AvroSource.java
index 4dd07261b..23346cb45 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/source/AvroSource.java
+++ b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/source/AvroSource.java
@@ -17,14 +17,21 @@
* under the License.
*/
-package org.apache.flume.source;
+package org.apache.flume.rpc.avro.source;
-import org.apache.avro.ipc.netty.NettyServer;
+import io.netty.channel.ChannelPipeline;
+import io.netty.handler.codec.compression.JZlibDecoder;
+import io.netty.handler.codec.compression.JZlibEncoder;
+import io.netty.handler.codec.compression.ZlibEncoder;
+import io.netty.handler.ipfilter.IpFilterRule;
+import io.netty.handler.ipfilter.IpFilterRuleType;
+import io.netty.handler.ipfilter.RuleBasedIpFilter;
+import io.netty.handler.ssl.SslHandler;
import org.apache.avro.ipc.Responder;
import org.apache.avro.ipc.Server;
+import org.apache.avro.ipc.netty.NettyServer;
import org.apache.avro.ipc.specific.SpecificResponder;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDrivenSource;
@@ -34,8 +41,10 @@ import org.apache.flume.conf.Configurable;
import org.apache.flume.conf.Configurables;
import org.apache.flume.conf.LogPrivacyUtil;
import org.apache.flume.event.EventBuilder;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.instrumentation.SourceCounter;
import org.apache.flume.netty.filter.PatternRule;
+import org.apache.flume.source.SslContextAwareAbstractSource;
import org.apache.flume.source.avro.AvroFlumeEvent;
import org.apache.flume.source.avro.AvroSourceProtocol;
import org.apache.flume.source.avro.Status;
@@ -43,7 +52,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLEngine;
-
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
@@ -54,15 +62,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
-import io.netty.channel.ChannelPipeline;
-import io.netty.handler.codec.compression.JZlibDecoder;
-import io.netty.handler.codec.compression.JZlibEncoder;
-import io.netty.handler.codec.compression.ZlibEncoder;
-import io.netty.handler.ipfilter.IpFilterRule;
-import io.netty.handler.ipfilter.IpFilterRuleType;
-import io.netty.handler.ipfilter.RuleBasedIpFilter;
-import io.netty.handler.ssl.SslHandler;
-
/**
* <p>
* A {@link Source} implementation that receives Avro events from clients that
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySource.java b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/source/SpoolDirectorySource.java
similarity index 97%
rename from flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySource.java
rename to flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/source/SpoolDirectorySource.java
index db4e0a32d..09f2b3c64 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySource.java
+++ b/flume-rpc/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/source/SpoolDirectorySource.java
@@ -15,23 +15,24 @@
* the License.
*/
-package org.apache.flume.source;
+package org.apache.flume.rpc.avro.source;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
-import org.apache.flume.ChannelException;
-import org.apache.flume.ChannelFullException;
+import org.apache.flume.exception.ChannelException;
+import org.apache.flume.exception.ChannelFullException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDrivenSource;
import org.apache.flume.FlumeException;
-import org.apache.flume.client.avro.ReliableSpoolingFileEventReader;
+import org.apache.flume.rpc.avro.serialization.ReliableSpoolingFileEventReader;
import org.apache.flume.conf.BatchSizeSupported;
import org.apache.flume.conf.Configurable;
import org.apache.flume.instrumentation.SourceCounter;
import org.apache.flume.serialization.DecodeErrorPolicy;
import org.apache.flume.serialization.LineDeserializer;
+import org.apache.flume.source.AbstractSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/flume-ng-sdk/src/test/java/org/apache/flume/api/TestFailoverRpcClient.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/api/TestFailoverRpcClient.java
similarity index 96%
rename from flume-ng-sdk/src/test/java/org/apache/flume/api/TestFailoverRpcClient.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/api/TestFailoverRpcClient.java
index c3eb205ab..593cb42ff 100644
--- a/flume-ng-sdk/src/test/java/org/apache/flume/api/TestFailoverRpcClient.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/api/TestFailoverRpcClient.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.api;
+package org.apache.flume.rpc.api;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
@@ -28,7 +28,10 @@ import org.apache.avro.ipc.Server;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.FlumeException;
-import org.apache.flume.api.RpcTestUtils.OKAvroHandler;
+import org.apache.flume.api.FailoverRpcClient;
+import org.apache.flume.api.RpcClientFactory;
+import org.apache.flume.rpc.avro.RpcTestUtils;
+import org.apache.flume.rpc.avro.RpcTestUtils.OKAvroHandler;
import org.apache.flume.event.EventBuilder;
import org.junit.Assert;
import org.junit.Test;
diff --git a/flume-ng-sdk/src/test/java/org/apache/flume/api/RpcTestUtils.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/RpcTestUtils.java
similarity index 98%
rename from flume-ng-sdk/src/test/java/org/apache/flume/api/RpcTestUtils.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/RpcTestUtils.java
index c9d0c7748..79bae3483 100644
--- a/flume-ng-sdk/src/test/java/org/apache/flume/api/RpcTestUtils.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/RpcTestUtils.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.api;
+package org.apache.flume.rpc.avro;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
@@ -31,7 +31,9 @@ import org.apache.avro.ipc.specific.SpecificResponder;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.FlumeException;
+import org.apache.flume.api.RpcClientConfigurationConstants;
import org.apache.flume.event.EventBuilder;
+import org.apache.flume.rpc.avro.client.NettyAvroRpcClient;
import org.apache.flume.source.avro.AvroFlumeEvent;
import org.apache.flume.source.avro.AvroSourceProtocol;
import org.apache.flume.source.avro.Status;
@@ -153,7 +155,7 @@ public class RpcTestUtils {
starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS, "h1");
starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + "h1",
"127.0.0.1" + ":" + port);
- NettyAvroRpcClient client = new NettyAvroRpcClient();
+ NettyAvroRpcClient client = NettyAvroRpcClient.newInstance();
client.configure(starterProp);
return client;
diff --git a/flume-ng-sdk/src/test/java/org/apache/flume/api/TestLoadBalancingRpcClient.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/TestLoadBalancingRpcClient.java
similarity index 98%
rename from flume-ng-sdk/src/test/java/org/apache/flume/api/TestLoadBalancingRpcClient.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/TestLoadBalancingRpcClient.java
index ba7a735cd..1bae722aa 100644
--- a/flume-ng-sdk/src/test/java/org/apache/flume/api/TestLoadBalancingRpcClient.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/TestLoadBalancingRpcClient.java
@@ -16,16 +16,19 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.api;
+package org.apache.flume.rpc.avro;
import junit.framework.Assert;
import org.apache.avro.ipc.Server;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.FlumeException;
-import org.apache.flume.api.RpcTestUtils.LoadBalancedAvroHandler;
-import org.apache.flume.api.RpcTestUtils.OKAvroHandler;
+import org.apache.flume.api.LoadBalancingRpcClient;
+import org.apache.flume.api.RpcClient;
+import org.apache.flume.api.RpcClientFactory;
import org.apache.flume.event.EventBuilder;
+import org.apache.flume.rpc.avro.RpcTestUtils.LoadBalancedAvroHandler;
+import org.apache.flume.rpc.avro.RpcTestUtils.OKAvroHandler;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/flume-ng-sdk/src/test/java/org/apache/flume/api/TestRpcClientFactory.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/TestRpcClientFactory.java
similarity index 96%
rename from flume-ng-sdk/src/test/java/org/apache/flume/api/TestRpcClientFactory.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/TestRpcClientFactory.java
index 2899925f8..9c42d7fb2 100644
--- a/flume-ng-sdk/src/test/java/org/apache/flume/api/TestRpcClientFactory.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/TestRpcClientFactory.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.api;
+package org.apache.flume.rpc.avro;
import java.nio.charset.Charset;
import java.util.ArrayList;
@@ -27,7 +27,9 @@ import org.apache.avro.ipc.Server;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.FlumeException;
-import org.apache.flume.api.RpcTestUtils.OKAvroHandler;
+import org.apache.flume.api.RpcClient;
+import org.apache.flume.api.RpcClientFactory;
+import org.apache.flume.rpc.avro.RpcTestUtils.OKAvroHandler;
import org.junit.Test;
import org.apache.flume.event.EventBuilder;
diff --git a/flume-ng-sdk/src/test/java/org/apache/flume/api/TestNettyAvroRpcClient.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/client/TestNettyAvroRpcClient.java
similarity index 97%
rename from flume-ng-sdk/src/test/java/org/apache/flume/api/TestNettyAvroRpcClient.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/client/TestNettyAvroRpcClient.java
index 6cd14548c..b6bd6aa02 100644
--- a/flume-ng-sdk/src/test/java/org/apache/flume/api/TestNettyAvroRpcClient.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/client/TestNettyAvroRpcClient.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.api;
+package org.apache.flume.rpc.avro.client;
import java.nio.charset.Charset;
import java.util.ArrayList;
@@ -27,10 +27,12 @@ import org.apache.avro.ipc.Server;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.FlumeException;
-import org.apache.flume.api.RpcTestUtils.FailedAvroHandler;
-import org.apache.flume.api.RpcTestUtils.OKAvroHandler;
-import org.apache.flume.api.RpcTestUtils.ThrowingAvroHandler;
-import org.apache.flume.api.RpcTestUtils.UnknownAvroHandler;
+import org.apache.flume.api.RpcClientConfigurationConstants;
+import org.apache.flume.rpc.avro.RpcTestUtils;
+import org.apache.flume.rpc.avro.RpcTestUtils.FailedAvroHandler;
+import org.apache.flume.rpc.avro.RpcTestUtils.OKAvroHandler;
+import org.apache.flume.rpc.avro.RpcTestUtils.ThrowingAvroHandler;
+import org.apache.flume.rpc.avro.RpcTestUtils.UnknownAvroHandler;
import org.apache.flume.event.EventBuilder;
import org.junit.Assert;
import org.junit.Test;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/serialization/SyslogAvroEventSerializer.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/SyslogAvroEventSerializer.java
similarity index 91%
rename from flume-ng-core/src/test/java/org/apache/flume/serialization/SyslogAvroEventSerializer.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/SyslogAvroEventSerializer.java
index 22e7e9e9b..a51bd665d 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/serialization/SyslogAvroEventSerializer.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/SyslogAvroEventSerializer.java
@@ -16,22 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.serialization;
+package org.apache.flume.rpc.avro.serialization;
import com.google.common.base.Charsets;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
import java.util.Map;
import org.apache.avro.Schema;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.conf.LogPrivacyUtil;
-import org.apache.flume.serialization.SyslogAvroEventSerializer.SyslogEvent;
+import org.apache.flume.rpc.avro.serialization.SyslogAvroEventSerializer.SyslogEvent;
+import org.apache.flume.serialization.EventSerializer;
import org.apache.flume.source.SyslogUtils;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,9 +52,9 @@ public class SyslogAvroEventSerializer
extends AbstractAvroEventSerializer<SyslogEvent> {
private static final DateTimeFormatter dateFmt1 =
- DateTimeFormat.forPattern("MMM dd HH:mm:ss").withZoneUTC();
+ DateTimeFormatter.ofPattern("MMM dd HH:mm:ss");
private static final DateTimeFormatter dateFmt2 =
- DateTimeFormat.forPattern("MMM d HH:mm:ss").withZoneUTC();
+ DateTimeFormatter.ofPattern("MMM d HH:mm:ss");
private static final Logger logger =
LoggerFactory.getLogger(SyslogAvroEventSerializer.class);
@@ -166,18 +168,18 @@ public class SyslogAvroEventSerializer
* recently.
*/
private static long parseRfc3164Date(String in) {
- DateTime date = null;
+ LocalDateTime date = null;
try {
- date = dateFmt1.parseDateTime(in);
- } catch (IllegalArgumentException e) {
+ date = LocalDateTime.parse(in, dateFmt1);
+ } catch (DateTimeParseException e) {
// ignore the exception, we act based on nullity of date object
logger.debug("Date parse failed on ({}), trying single-digit date", in);
}
if (date == null) {
try {
- date = dateFmt2.parseDateTime(in);
- } catch (IllegalArgumentException e) {
+ date = LocalDateTime.parse(in, dateFmt2);
+ } catch (DateTimeParseException e) {
// ignore the exception, we act based on nullity of date object
logger.debug("2nd date parse failed on ({}), unknown date format", in);
}
@@ -187,9 +189,9 @@ public class SyslogAvroEventSerializer
// rfc3164 dates are really dumb.
// NB: cannot handle replaying of old logs or going back to the future
if (date != null) {
- DateTime now = new DateTime();
+ LocalDateTime now = LocalDateTime.now();
int year = now.getYear();
- DateTime corrected = date.withYear(year);
+ LocalDateTime corrected = date.withYear(year);
// flume clock is ahead or there is some latency, and the year rolled
if (corrected.isAfter(now) && corrected.minusMonths(1).isAfter(now)) {
@@ -205,7 +207,7 @@ public class SyslogAvroEventSerializer
return 0;
}
- return date.getMillis();
+ return date.toInstant(ZoneOffset.UTC).toEpochMilli();
}
public static class Builder implements EventSerializer.Builder {
diff --git a/flume-ng-core/src/test/java/org/apache/flume/serialization/TestAvroEventDeserializer.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestAvroEventDeserializer.java
similarity index 95%
rename from flume-ng-core/src/test/java/org/apache/flume/serialization/TestAvroEventDeserializer.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestAvroEventDeserializer.java
index b95433fd1..b57fdc967 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/serialization/TestAvroEventDeserializer.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestAvroEventDeserializer.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.flume.serialization;
+package org.apache.flume.rpc.avro.serialization;
import org.apache.avro.Schema;
import org.apache.avro.SchemaNormalization;
@@ -30,6 +30,10 @@ import org.apache.avro.io.DecoderFactory;
import org.apache.commons.codec.binary.Hex;
import org.apache.flume.Context;
import org.apache.flume.Event;
+import org.apache.flume.serialization.EventDeserializer;
+import org.apache.flume.serialization.ResettableFileInputStream;
+import org.apache.flume.serialization.ResettableInputStream;
+import org.apache.flume.serialization.TransientPositionTracker;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/serialization/TestDurablePositionTracker.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestDurablePositionTracker.java
similarity index 96%
rename from flume-ng-core/src/test/java/org/apache/flume/serialization/TestDurablePositionTracker.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestDurablePositionTracker.java
index 85edaa3e3..3024f3077 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/serialization/TestDurablePositionTracker.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestDurablePositionTracker.java
@@ -15,10 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.flume.serialization;
+package org.apache.flume.rpc.avro.serialization;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
+import org.apache.flume.serialization.PositionTracker;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/serialization/TestFlumeEventAvroEventSerializer.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestFlumeEventAvroEventSerializer.java
similarity index 97%
rename from flume-ng-core/src/test/java/org/apache/flume/serialization/TestFlumeEventAvroEventSerializer.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestFlumeEventAvroEventSerializer.java
index ded3b1353..498c413aa 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/serialization/TestFlumeEventAvroEventSerializer.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestFlumeEventAvroEventSerializer.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.serialization;
+package org.apache.flume.rpc.avro.serialization;
import com.google.common.base.Charsets;
import org.apache.avro.file.DataFileReader;
@@ -27,6 +27,7 @@ import org.apache.avro.io.DatumReader;
import org.apache.commons.io.FileUtils;
import org.apache.flume.Context;
import org.apache.flume.event.EventBuilder;
+import org.apache.flume.serialization.EventSerializer;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Test;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestReliableSpoolingFileEventReader.java
similarity index 95%
rename from flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestReliableSpoolingFileEventReader.java
index 0d448247f..0c94990a0 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestReliableSpoolingFileEventReader.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.flume.client.avro;
+package org.apache.flume.rpc.avro.serialization;
import java.io.File;
import java.io.FileFilter;
@@ -42,9 +42,8 @@ import org.apache.commons.lang.SystemUtils;
import org.apache.commons.lang.mutable.MutableLong;
import org.apache.flume.Event;
import org.apache.flume.FlumeException;
-import org.apache.flume.client.avro.ReliableSpoolingFileEventReader.DeletePolicy;
+import org.apache.flume.event.ReliableEventReader;
import org.apache.flume.instrumentation.SourceCounter;
-import org.apache.flume.client.avro.ReliableSpoolingFileEventReader.TrackingPolicy;
import org.apache.flume.source.SpoolDirectorySourceConfigurationConstants;
import org.apache.flume.source.SpoolDirectorySourceConfigurationConstants.ConsumeOrder;
import org.junit.After;
@@ -158,7 +157,7 @@ public class TestReliableSpoolingFileEventReader {
ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder()
.spoolDirectory(WORK_DIR)
.includePattern("^file2$")
- .deletePolicy(DeletePolicy.IMMEDIATE.toString())
+ .deletePolicy(ReliableSpoolingFileEventReader.DeletePolicy.IMMEDIATE.toString())
.sourceCounter(new SourceCounter("test"))
.build();
@@ -180,7 +179,7 @@ public class TestReliableSpoolingFileEventReader {
new ReliableSpoolingFileEventReader.Builder()
.spoolDirectory(WORK_DIR)
.ignorePattern("^file2$")
- .deletePolicy(DeletePolicy.IMMEDIATE.toString())
+ .deletePolicy(ReliableSpoolingFileEventReader.DeletePolicy.IMMEDIATE.toString())
.sourceCounter(new SourceCounter("test"))
.build();
@@ -210,7 +209,7 @@ public class TestReliableSpoolingFileEventReader {
.spoolDirectory(WORK_DIR)
.ignorePattern("^file[013]$")
.includePattern("^file2$")
- .deletePolicy(DeletePolicy.IMMEDIATE.toString())
+ .deletePolicy(ReliableSpoolingFileEventReader.DeletePolicy.IMMEDIATE.toString())
.sourceCounter(new SourceCounter("test"))
.build();
@@ -238,7 +237,7 @@ public class TestReliableSpoolingFileEventReader {
.spoolDirectory(WORK_DIR)
.ignorePattern("^file2$")
.includePattern("^file2$")
- .deletePolicy(DeletePolicy.IMMEDIATE.toString())
+ .deletePolicy(ReliableSpoolingFileEventReader.DeletePolicy.IMMEDIATE.toString())
.sourceCounter(new SourceCounter("test"))
.build();
@@ -307,7 +306,7 @@ public class TestReliableSpoolingFileEventReader {
public void testFileDeletion() throws IOException {
ReliableEventReader reader =
new ReliableSpoolingFileEventReader.Builder().spoolDirectory(WORK_DIR)
- .deletePolicy(DeletePolicy.IMMEDIATE.name())
+ .deletePolicy(ReliableSpoolingFileEventReader.DeletePolicy.IMMEDIATE.name())
.sourceCounter(new SourceCounter("test"))
.build();
@@ -333,7 +332,7 @@ public class TestReliableSpoolingFileEventReader {
File workDir = WORK_DIR;
if(workDir.setReadOnly()){
new ReliableSpoolingFileEventReader.Builder().spoolDirectory(workDir)
- .trackingPolicy(TrackingPolicy.RENAME.name())
+ .trackingPolicy(ReliableSpoolingFileEventReader.TrackingPolicy.RENAME.name())
.sourceCounter(new SourceCounter("test"))
.build();
} else {
@@ -353,7 +352,7 @@ public class TestReliableSpoolingFileEventReader {
}
if(workDir.setReadOnly()){
new ReliableSpoolingFileEventReader.Builder().spoolDirectory(workDir)
- .trackingPolicy(TrackingPolicy.TRACKER_DIR.name())
+ .trackingPolicy(ReliableSpoolingFileEventReader.TrackingPolicy.TRACKER_DIR.name())
.trackerDirPath(trackerDirPath)
.sourceCounter(new SourceCounter("test"))
.build();
@@ -550,29 +549,29 @@ public class TestReliableSpoolingFileEventReader {
}
@Test public void testLargeNumberOfFilesOLDEST() throws IOException {
- templateTestForRecursiveDirs(ConsumeOrder.OLDEST, null, 3, 3, 37, TrackingPolicy.RENAME);
+ templateTestForRecursiveDirs(ConsumeOrder.OLDEST, null, 3, 3, 37, ReliableSpoolingFileEventReader.TrackingPolicy.RENAME);
}
@Test public void testLargeNumberOfFilesYOUNGEST() throws IOException {
templateTestForRecursiveDirs(ConsumeOrder.YOUNGEST, Comparator.reverseOrder(),
- 3, 3, 37, TrackingPolicy.RENAME);
+ 3, 3, 37, ReliableSpoolingFileEventReader.TrackingPolicy.RENAME);
}
@Test public void testLargeNumberOfFilesRANDOM() throws IOException {
- templateTestForRecursiveDirs(ConsumeOrder.RANDOM, null, 3, 3, 37, TrackingPolicy.RENAME);
+ templateTestForRecursiveDirs(ConsumeOrder.RANDOM, null, 3, 3, 37, ReliableSpoolingFileEventReader.TrackingPolicy.RENAME);
}
@Test public void testLargeNumberOfFilesOLDESTTrackerDir() throws IOException {
- templateTestForRecursiveDirs(ConsumeOrder.OLDEST, null, 3, 3, 10, TrackingPolicy.TRACKER_DIR);
+ templateTestForRecursiveDirs(ConsumeOrder.OLDEST, null, 3, 3, 10, ReliableSpoolingFileEventReader.TrackingPolicy.TRACKER_DIR);
}
@Test public void testLargeNumberOfFilesYOUNGESTTrackerDir() throws IOException {
templateTestForRecursiveDirs(ConsumeOrder.YOUNGEST, Comparator.reverseOrder(),
- 3, 3, 10, TrackingPolicy.TRACKER_DIR);
+ 3, 3, 10, ReliableSpoolingFileEventReader.TrackingPolicy.TRACKER_DIR);
}
@Test public void testLargeNumberOfFilesRANDOMTrackerDir() throws IOException {
- templateTestForRecursiveDirs(ConsumeOrder.RANDOM, null, 3, 3, 10, TrackingPolicy.TRACKER_DIR);
+ templateTestForRecursiveDirs(ConsumeOrder.RANDOM, null, 3, 3, 10, ReliableSpoolingFileEventReader.TrackingPolicy.TRACKER_DIR);
}
@Test
@@ -607,7 +606,7 @@ public class TestReliableSpoolingFileEventReader {
}
private void templateTestForRecursiveDirs(ConsumeOrder order, Comparator<Long> comparator, int depth, int dirNum,
- int fileNum, TrackingPolicy trackingPolicy) throws IOException {
+ int fileNum, ReliableSpoolingFileEventReader.TrackingPolicy trackingPolicy) throws IOException {
File dir = null;
try {
dir = new File("target/test/work/" + this.getClass().getSimpleName() + "_large");
diff --git a/flume-ng-core/src/test/java/org/apache/flume/serialization/TestResettableFileInputStream.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestResettableFileInputStream.java
similarity index 98%
rename from flume-ng-core/src/test/java/org/apache/flume/serialization/TestResettableFileInputStream.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestResettableFileInputStream.java
index 41cdf0b8e..149c2139c 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/serialization/TestResettableFileInputStream.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestResettableFileInputStream.java
@@ -15,12 +15,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.flume.serialization;
+package org.apache.flume.rpc.avro.serialization;
import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
import junit.framework.Assert;
+import org.apache.flume.serialization.DecodeErrorPolicy;
+import org.apache.flume.serialization.PositionTracker;
+import org.apache.flume.serialization.ResettableFileInputStream;
+import org.apache.flume.serialization.ResettableInputStream;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestSpoolingFileLineReader.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestSpoolingFileLineReader.java
similarity index 99%
rename from flume-ng-core/src/test/java/org/apache/flume/client/avro/TestSpoolingFileLineReader.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestSpoolingFileLineReader.java
index 3d43bf4f8..8015a3d42 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestSpoolingFileLineReader.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestSpoolingFileLineReader.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.flume.client.avro;
+package org.apache.flume.rpc.avro.serialization;
import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/serialization/TestSyslogAvroEventSerializer.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestSyslogAvroEventSerializer.java
similarity index 96%
rename from flume-ng-core/src/test/java/org/apache/flume/serialization/TestSyslogAvroEventSerializer.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestSyslogAvroEventSerializer.java
index ba9f4ab73..bf6f3dc70 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/serialization/TestSyslogAvroEventSerializer.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/serialization/TestSyslogAvroEventSerializer.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.serialization;
+package org.apache.flume.rpc.avro.serialization;
import java.io.File;
import java.io.FileNotFoundException;
@@ -34,6 +34,8 @@ import org.apache.commons.io.FileUtils;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.event.EventBuilder;
+import org.apache.flume.serialization.EventSerializer;
+import org.apache.flume.serialization.EventSerializerFactory;
import org.apache.flume.source.SyslogUtils;
import org.junit.Assert;
import org.junit.Assume;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/sink/TestAvroSink.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/sink/TestAvroSink.java
similarity index 99%
rename from flume-ng-core/src/test/java/org/apache/flume/sink/TestAvroSink.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/sink/TestAvroSink.java
index 30bc5f6b5..50722eafe 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/sink/TestAvroSink.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/sink/TestAvroSink.java
@@ -17,14 +17,13 @@
* under the License.
*/
-package org.apache.flume.sink;
+package org.apache.flume.rpc.avro.sink;
import com.google.common.base.Charsets;
import org.apache.avro.ipc.netty.NettyServer;
import org.apache.avro.ipc.Server;
import org.apache.avro.ipc.specific.SpecificResponder;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
import org.apache.flume.ChannelSelector;
import org.apache.flume.Context;
import org.apache.flume.Event;
@@ -38,10 +37,11 @@ import org.apache.flume.channel.MemoryChannel;
import org.apache.flume.channel.ReplicatingChannelSelector;
import org.apache.flume.conf.Configurables;
import org.apache.flume.event.EventBuilder;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.instrumentation.SinkCounter;
import org.apache.flume.lifecycle.LifecycleController;
import org.apache.flume.lifecycle.LifecycleState;
-import org.apache.flume.source.AvroSource;
+import org.apache.flume.rpc.avro.source.AvroSource;
import org.apache.flume.source.avro.AvroFlumeEvent;
import org.apache.flume.source.avro.AvroSourceProtocol;
import org.apache.flume.source.avro.Status;
diff --git a/flume-ng-core/src/test/java/org/apache/flume/sink/TestDefaultSinkFactory.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/sink/TestDefaultSinkFactory.java
similarity index 91%
copy from flume-ng-core/src/test/java/org/apache/flume/sink/TestDefaultSinkFactory.java
copy to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/sink/TestDefaultSinkFactory.java
index cf6cbbcc7..fd726a471 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/sink/TestDefaultSinkFactory.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/sink/TestDefaultSinkFactory.java
@@ -17,10 +17,14 @@
* under the License.
*/
-package org.apache.flume.sink;
+package org.apache.flume.rpc.avro.sink;
import org.apache.flume.Sink;
import org.apache.flume.SinkFactory;
+import org.apache.flume.sink.DefaultSinkFactory;
+import org.apache.flume.sink.LoggerSink;
+import org.apache.flume.sink.NullSink;
+import org.apache.flume.sink.RollingFileSink;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializerType.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/source/MockSource.java
similarity index 53%
copy from flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializerType.java
copy to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/source/MockSource.java
index 86d243fe1..a6a3c71db 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializerType.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/source/MockSource.java
@@ -16,27 +16,48 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.serialization;
+package org.apache.flume.rpc.avro.source;
-import org.apache.flume.annotations.InterfaceAudience;
-import org.apache.flume.annotations.InterfaceStability;
+import org.apache.flume.Source;
+import org.apache.flume.channel.ChannelProcessor;
+import org.apache.flume.lifecycle.LifecycleState;
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public enum EventSerializerType {
- TEXT(BodyTextEventSerializer.Builder.class),
- HEADER_AND_TEXT(HeaderAndBodyTextEventSerializer.Builder.class),
- AVRO_EVENT(FlumeEventAvroEventSerializer.Builder.class),
- OTHER(null);
+public class MockSource implements Source {
- private final Class<? extends EventSerializer.Builder> builderClass;
+ private String name;
- EventSerializerType(Class<? extends EventSerializer.Builder> builderClass) {
- this.builderClass = builderClass;
+ public MockSource() {
}
- public Class<? extends EventSerializer.Builder> getBuilderClass() {
- return builderClass;
+ @Override
+ public void start() {
}
+ @Override
+ public void stop() {
+ }
+
+ @Override
+ public LifecycleState getLifecycleState() {
+ return null;
+ }
+
+ @Override
+ public void setChannelProcessor(ChannelProcessor cp) {
+ }
+
+ @Override
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public ChannelProcessor getChannelProcessor() {
+ return null;
+ }
}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestDefaultSourceFactory.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/source/TestDefaultSourceFactory.java
similarity index 87%
copy from flume-ng-core/src/test/java/org/apache/flume/source/TestDefaultSourceFactory.java
copy to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/source/TestDefaultSourceFactory.java
index 97e024f87..2a386d164 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestDefaultSourceFactory.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/source/TestDefaultSourceFactory.java
@@ -17,10 +17,18 @@
* under the License.
*/
-package org.apache.flume.source;
+package org.apache.flume.rpc.avro.source;
import org.apache.flume.Source;
import org.apache.flume.SourceFactory;
+import org.apache.flume.source.DefaultSourceFactory;
+import org.apache.flume.source.ExecSource;
+import org.apache.flume.source.MultiportSyslogTCPSource;
+import org.apache.flume.source.NetcatSource;
+import org.apache.flume.source.NetcatUdpSource;
+import org.apache.flume.source.SequenceGeneratorSource;
+import org.apache.flume.source.SyslogTcpSource;
+import org.apache.flume.source.SyslogUDPSource;
import org.apache.flume.source.http.HTTPSource;
import org.junit.Assert;
import org.junit.Before;
@@ -76,7 +84,6 @@ public class TestDefaultSourceFactory {
verifySourceCreation("spooldir-src", "spooldir",
SpoolDirectorySource.class);
verifySourceCreation("http-src", "http", HTTPSource.class);
- verifySourceCreation("thrift-src", "thrift", ThriftSource.class);
verifySourceCreation("custom-src", MockSource.class.getCanonicalName(),
MockSource.class);
}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestSpoolDirectorySource.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/source/TestSpoolDirectorySource.java
similarity index 98%
rename from flume-ng-core/src/test/java/org/apache/flume/source/TestSpoolDirectorySource.java
rename to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/source/TestSpoolDirectorySource.java
index 7c671a648..db9cd1983 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestSpoolDirectorySource.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/source/TestSpoolDirectorySource.java
@@ -15,14 +15,14 @@
* the License.
*/
-package org.apache.flume.source;
+package org.apache.flume.rpc.avro.source;
import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
-import org.apache.flume.ChannelFullException;
+import org.apache.flume.exception.ChannelException;
+import org.apache.flume.exception.ChannelFullException;
import org.apache.flume.ChannelSelector;
import org.apache.flume.Context;
import org.apache.flume.Event;
@@ -30,11 +30,12 @@ import org.apache.flume.Transaction;
import org.apache.flume.channel.ChannelProcessor;
import org.apache.flume.channel.MemoryChannel;
import org.apache.flume.channel.ReplicatingChannelSelector;
-import org.apache.flume.client.avro.ReliableSpoolingFileEventReader;
import org.apache.flume.conf.Configurables;
import org.apache.flume.instrumentation.SourceCounter;
import org.apache.flume.lifecycle.LifecycleController;
import org.apache.flume.lifecycle.LifecycleState;
+import org.apache.flume.rpc.avro.serialization.ReliableSpoolingFileEventReader;
+import org.apache.flume.source.SpoolDirectorySourceConfigurationConstants;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
diff --git a/flume-ng-node/src/test/java/org/apache/flume/node/TestAbstractConfigurationProvider.java b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/node/TestAbstractConfigurationProvider.java
similarity index 98%
copy from flume-ng-node/src/test/java/org/apache/flume/node/TestAbstractConfigurationProvider.java
copy to flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/node/TestAbstractConfigurationProvider.java
index 7810c5bca..8c16e7268 100644
--- a/flume-ng-node/src/test/java/org/apache/flume/node/TestAbstractConfigurationProvider.java
+++ b/flume-rpc/flume-rpc-avro/src/test/java/org/apache/flume/rpc/node/TestAbstractConfigurationProvider.java
@@ -15,12 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.flume.node;
+package org.apache.flume.rpc.node;
import com.google.common.collect.Maps;
import junit.framework.Assert;
import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
@@ -30,6 +29,9 @@ import org.apache.flume.annotations.Recyclable;
import org.apache.flume.channel.AbstractChannel;
import org.apache.flume.conf.Configurable;
import org.apache.flume.conf.FlumeConfiguration;
+import org.apache.flume.exception.ChannelException;
+import org.apache.flume.node.AbstractConfigurationProvider;
+import org.apache.flume.node.MaterializedConfiguration;
import org.apache.flume.sink.AbstractSink;
import org.apache.flume.source.AbstractSource;
import org.junit.Test;
diff --git a/flume-ng-core/src/test/resources/TestResettableFileInputStream_1.avro b/flume-rpc/flume-rpc-avro/src/test/resources/TestResettableFileInputStream_1.avro
similarity index 100%
rename from flume-ng-core/src/test/resources/TestResettableFileInputStream_1.avro
rename to flume-rpc/flume-rpc-avro/src/test/resources/TestResettableFileInputStream_1.avro
diff --git a/flume-ng-core/src/test/resources/TestResettableFileInputStream_1.truncated.avro b/flume-rpc/flume-rpc-avro/src/test/resources/TestResettableFileInputStream_1.truncated.avro
similarity index 100%
rename from flume-ng-core/src/test/resources/TestResettableFileInputStream_1.truncated.avro
rename to flume-rpc/flume-rpc-avro/src/test/resources/TestResettableFileInputStream_1.truncated.avro
diff --git a/flume-rpc/flume-rpc-avro/src/test/resources/certs/gencerts.sh b/flume-rpc/flume-rpc-avro/src/test/resources/certs/gencerts.sh
new file mode 100755
index 000000000..0fb0263de
--- /dev/null
+++ b/flume-rpc/flume-rpc-avro/src/test/resources/certs/gencerts.sh
@@ -0,0 +1,31 @@
+mkdir tmp
+rm ../truststorefile.jks
+rm ../keystorefile.jks
+rm ../server.flume-keystore.p12
+# Create the CA key and certificate
+openssl req -config rootca.conf -new -x509 -nodes -keyout tmp/flume-cacert.key -out tmp/flume-ca.crt -days 10960
+# Create the trust store and import the certificate
+keytool -keystore ../truststorefile.jks -storetype jks -importcert -file 'tmp/flume-ca.crt' -keypass password -storepass password -alias flume-cacert -noprompt
+#Import the root certificate
+keytool -keystore ../keystorefile.jks -alias flume-ca -importcert -file tmp/flume-ca.crt -keypass password -storepass password -noprompt
+# Create the client private key in the client key store
+keytool -genkeypair -keyalg RSA -alias client -keystore ../keystorefile.jks -storepass password -keypass password -validity 10960 -keysize 2048 -dname "CN=client.flume, C=US"
+# Create a signing request for the client #
+keytool -keystore ../keystorefile.jks -alias client -certreq -file tmp/client.csr -keypass password -storepass password
+# Sign the client certificate
+openssl x509 -req -CA 'tmp/flume-ca.crt' -CAkey 'tmp/flume-cacert.key' -in tmp/client.csr -out tmp/client.crt_signed -days 10960 -CAcreateserial -passin pass:password
+# Verify the signed certificate
+openssl verify -CAfile 'tmp/flume-ca.crt' tmp/client.crt_signed
+#Import the client's signed certificate
+keytool -keystore ../keystorefile.jks -alias client -importcert -file tmp/client.crt_signed -keypass password -storepass password -noprompt
+#Verify the keystore
+keytool -list -v -keystore ../keystorefile.jks -storepass password
+# Create the server private key in the server key store
+keytool -genkeypair -keyalg RSA -alias server -keystore ../server.flume-keystore.p12 -storepass password -storetype PKCS12 -keypass password -validity 10960 -keysize 2048 -dname "CN=server.flume, C=US"
+# Create a signing request for the server #
+keytool -keystore ../server.flume-keystore.p12 -alias server -certreq -file tmp/server.csr -keypass password -storepass password
+# Sign the server certificate
+openssl x509 -req -CA 'tmp/flume-ca.crt' -CAkey 'tmp/flume-cacert.key' -in tmp/server.csr -out ../server.flume-crt.pem -days 10960 -CAcreateserial -passin pass:password
+# Extract the private key
+openssl pkcs12 -in ../server.flume-keystore.p12 -passin pass:password -nokeys -out ../server.flume.pem
+rm -rf tmp
diff --git a/flume-rpc/flume-rpc-avro/src/test/resources/certs/rootca.conf b/flume-rpc/flume-rpc-avro/src/test/resources/certs/rootca.conf
new file mode 100644
index 000000000..722e9c39b
--- /dev/null
+++ b/flume-rpc/flume-rpc-avro/src/test/resources/certs/rootca.conf
@@ -0,0 +1,9 @@
+[ req ]
+distinguished_name = CA_DN
+prompt = no
+output_password = password
+default_bits = 2048
+
+[ CA_DN ]
+C = US
+CN = flume-ca
diff --git a/flume-rpc/flume-rpc-avro/src/test/resources/certs/server.conf b/flume-rpc/flume-rpc-avro/src/test/resources/certs/server.conf
new file mode 100644
index 000000000..7a9fb5835
--- /dev/null
+++ b/flume-rpc/flume-rpc-avro/src/test/resources/certs/server.conf
@@ -0,0 +1,9 @@
+[ req ]
+distinguished_name = CA_DN
+prompt = no
+output_password = password
+default_bits = 2048
+
+[ CA_DN ]
+C = US
+CN = server.flume
diff --git a/flume-rpc/flume-rpc-avro/src/test/resources/jettykeystore b/flume-rpc/flume-rpc-avro/src/test/resources/jettykeystore
new file mode 100644
index 000000000..db76bcbc0
Binary files /dev/null and b/flume-rpc/flume-rpc-avro/src/test/resources/jettykeystore differ
diff --git a/flume-rpc/flume-rpc-avro/src/test/resources/keystorefile.jks b/flume-rpc/flume-rpc-avro/src/test/resources/keystorefile.jks
new file mode 100644
index 000000000..d62e95792
Binary files /dev/null and b/flume-rpc/flume-rpc-avro/src/test/resources/keystorefile.jks differ
diff --git a/flume-ng-tests/src/test/resources/log4j2.xml b/flume-rpc/flume-rpc-avro/src/test/resources/log4j2.xml
similarity index 93%
copy from flume-ng-tests/src/test/resources/log4j2.xml
copy to flume-rpc/flume-rpc-avro/src/test/resources/log4j2.xml
index a6e2e30bd..7caf0d2a7 100644
--- a/flume-ng-tests/src/test/resources/log4j2.xml
+++ b/flume-rpc/flume-rpc-avro/src/test/resources/log4j2.xml
@@ -23,8 +23,8 @@
</Console>
</Appenders>
<Loggers>
+ <Logger name="org.apache.flume" level="INFO"/>
<Root level="INFO">
- <Logger name="org.apache.flume" level="DEBUG"/>
<AppenderRef ref="Console" />
</Root>
</Loggers>
diff --git a/flume-rpc/flume-rpc-avro/src/test/resources/server.flume-crt.pem b/flume-rpc/flume-rpc-avro/src/test/resources/server.flume-crt.pem
new file mode 100644
index 000000000..9e3774427
--- /dev/null
+++ b/flume-rpc/flume-rpc-avro/src/test/resources/server.flume-crt.pem
@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICwjCCAaoCCQD8AQyla5FbDjANBgkqhkiG9w0BAQsFADAgMQswCQYDVQQGEwJV
+UzERMA8GA1UEAwwIZmx1bWUtY2EwIBcNMjMwMzIyMDAyODM0WhgPMjA1MzAzMjQw
+MDI4MzRaMCQxCzAJBgNVBAYTAlVTMRUwEwYDVQQDEwxzZXJ2ZXIuZmx1bWUwggEi
+MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEEAUShlX2lWDYiEgcZL1JzWFw
+auNkdxOGo1DtD3YaZdrU1GS2UwspLm0qcSFEF1Sx6uVTABdgjZGnxJLSTJkabVu/
+iMuP+EQrE/4AWJoyuuaYMSG0EGeP+mttTtLHYyt/k9NJkABYxFZkNqdDuo+lF/Vs
+QdW3icym0vqMwiIvE+VKw9j3F+zFZizfx6MiBH0uuNrXHFCNUg52/cbeyiXO1mks
+yruOV+PF8/44zAepjLWiJgp7Wo6ejXmLvR+k68RwB5V7fXrzPYueM9GQDmLffkdO
+ZhrcafiRFGlzWSmC820Eb2b5+cVnm96XAlUXE5ao5o58oMmcufMnJ5k2UbFJAgMB
+AAEwDQYJKoZIhvcNAQELBQADggEBAHBGpqraT39aw/HVrJdmpsw8CwSmdiir+NYk
+5PprbIKAyf/P/9ObKcqesO8d8CQZVvzzm+Ok2rgcALDIl/TbbAVUPizIrN4AiH+Z
+BPOqDFF4taWkw73iMDiq61QS8SpJIOxxmL8PsK5eefuABrpumnVgW5X9BT/uMIqW
+NOwiyII3NVvtlErcdAL/ZTYWc3S8CEWVRc88ZpIBSLB4/tqQbPM+m+ZtYMSKi3Sh
+ugOjonPuteeQqu6R7HRYOajepKdGe048Moq90v0IrDI8v+rbezLFpEWOnG0fUDEq
+LfA9l7e/q1ukXRW4ccJWZWXLrZbEbX5hJeTlyYhHciwB5jfueMM=
+-----END CERTIFICATE-----
diff --git a/flume-rpc/flume-rpc-avro/src/test/resources/server.flume-keystore.p12 b/flume-rpc/flume-rpc-avro/src/test/resources/server.flume-keystore.p12
new file mode 100644
index 000000000..da51f355d
Binary files /dev/null and b/flume-rpc/flume-rpc-avro/src/test/resources/server.flume-keystore.p12 differ
diff --git a/flume-rpc/flume-rpc-avro/src/test/resources/server.flume.pem b/flume-rpc/flume-rpc-avro/src/test/resources/server.flume.pem
new file mode 100644
index 000000000..6bc71ed79
--- /dev/null
+++ b/flume-rpc/flume-rpc-avro/src/test/resources/server.flume.pem
@@ -0,0 +1,23 @@
+Bag Attributes
+ friendlyName: server
+ localKeyID: 54 69 6D 65 20 31 36 37 39 34 34 34 39 31 33 33 33 39
+subject=/C=US/CN=server.flume
+issuer=/C=US/CN=server.flume
+-----BEGIN CERTIFICATE-----
+MIIC6TCCAdGgAwIBAgIEMlDgqzANBgkqhkiG9w0BAQsFADAkMQswCQYDVQQGEwJV
+UzEVMBMGA1UEAxMMc2VydmVyLmZsdW1lMCAXDTIzMDMyMjAwMjgzM1oYDzIwNTMw
+MzI0MDAyODMzWjAkMQswCQYDVQQGEwJVUzEVMBMGA1UEAxMMc2VydmVyLmZsdW1l
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxBAFEoZV9pVg2IhIHGS9
+Sc1hcGrjZHcThqNQ7Q92GmXa1NRktlMLKS5tKnEhRBdUserlUwAXYI2Rp8SS0kyZ
+Gm1bv4jLj/hEKxP+AFiaMrrmmDEhtBBnj/prbU7Sx2Mrf5PTSZAAWMRWZDanQ7qP
+pRf1bEHVt4nMptL6jMIiLxPlSsPY9xfsxWYs38ejIgR9Lrja1xxQjVIOdv3G3sol
+ztZpLMq7jlfjxfP+OMwHqYy1oiYKe1qOno15i70fpOvEcAeVe3168z2LnjPRkA5i
+335HTmYa3Gn4kRRpc1kpgvNtBG9m+fnFZ5velwJVFxOWqOaOfKDJnLnzJyeZNlGx
+SQIDAQABoyEwHzAdBgNVHQ4EFgQUClD6FZ+qPIFrtBaz0swRTtWt1WEwDQYJKoZI
+hvcNAQELBQADggEBALuX9E+tRWvvA9uULj9Iq+k9iUNMQzkmyzXGu7hY46ZU9lx+
+fNnLZq82zz9rHq8IhK4HsLIsPCLfNeXwG/TNR4zUHKI53lzkburxgu76soMUDbHX
+8udyUgrs0YjQcppw6IOOmlZtgNeF2nu7jeoXrCaA07yXzehAqukHv7glWaV3oORc
+rDkZvHfJ2G7hPbUYYIeouJsbG9rNukNPOY9JEYGFYzDxZ8hlFt7Lp/icbdpjFGDV
+tkMtPpVz59B47j/Kk/k5zxaDLnD42svL8GByyM5UxvAqAlYnfMKiZqXfY0JbCpMC
+e9Z5xOyt9F8NLFyjRsmBlJD61LuLb8hAZK/Ho70=
+-----END CERTIFICATE-----
diff --git a/flume-rpc/flume-rpc-avro/src/test/resources/server.p12 b/flume-rpc/flume-rpc-avro/src/test/resources/server.p12
new file mode 100644
index 000000000..07eef5168
Binary files /dev/null and b/flume-rpc/flume-rpc-avro/src/test/resources/server.p12 differ
diff --git a/flume-ng-core/src/test/resources/syslog_event.avsc b/flume-rpc/flume-rpc-avro/src/test/resources/syslog_event.avsc
similarity index 100%
rename from flume-ng-core/src/test/resources/syslog_event.avsc
rename to flume-rpc/flume-rpc-avro/src/test/resources/syslog_event.avsc
diff --git a/flume-rpc/flume-rpc-avro/src/test/resources/truststore.jks b/flume-rpc/flume-rpc-avro/src/test/resources/truststore.jks
new file mode 100644
index 000000000..8c08349e0
Binary files /dev/null and b/flume-rpc/flume-rpc-avro/src/test/resources/truststore.jks differ
diff --git a/flume-rpc/flume-rpc-avro/src/test/resources/truststorefile.jks b/flume-rpc/flume-rpc-avro/src/test/resources/truststorefile.jks
new file mode 100644
index 000000000..a0c3a49ac
Binary files /dev/null and b/flume-rpc/flume-rpc-avro/src/test/resources/truststorefile.jks differ
diff --git a/flume-rpc/flume-rpc-thrift/pom.xml b/flume-rpc/flume-rpc-thrift/pom.xml
new file mode 100644
index 000000000..e73e7eda7
--- /dev/null
+++ b/flume-rpc/flume-rpc-thrift/pom.xml
@@ -0,0 +1,216 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-rpc</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-rpc-thrift</artifactId>
+ <name>Flume RPC Thrift</name>
+
+ <properties>
+ <thrift.version>0.14.2</thrift.version>
+ <module.name>org.apache.flume.rpc.thrift</module.name>
+ <spotbugs.maxAllowedViolations>44</spotbugs.maxAllowedViolations>
+ <pmd.maxAllowedViolations>121</pmd.maxAllowedViolations>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-sdk</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-auth</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ <version>${thrift.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <scm>
+ <url>https://gitbox.apache.org/repos/asf/flume-spring-boot.git</url>
+ <developerConnection>https://gitbox.apache.org/repos/asf/flume-spring-boot.git</developerConnection>
+ <connection>https://gitbox.apache.org/repos/asf/flume-spring-boot.git</connection>
+ </scm>
+
+ <developers>
+ <developer>
+ <name>Ralph Goers</name>
+ <id>rgoers</id>
+ <email>rgoers@apache.org</email>
+ <organization>Nextiva</organization>
+ </developer>
+ </developers>
+
+ <organization>
+ <name>Apache Software Foundation</name>
+ <url>http://www.apache.org</url>
+ </organization>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${rat.version}</version>
+ <configuration>
+ <excludes>
+ <exclude>**/.idea/</exclude>
+ <exclude>**/*.iml</exclude>
+ <exclude>src/main/resources/META-INF/services/**/*</exclude>
+ <exclude>**/nb-configuration.xml</exclude>
+ <exclude>.git/</exclude>
+ <exclude>patchprocess/</exclude>
+ <exclude>.gitignore</exclude>
+ <exclude>**/*.yml</exclude>
+ <exclude>**/*.yaml</exclude>
+ <exclude>**/*.json</exclude>
+ <!-- ASF jenkins box puts the Maven repo in our root directory. -->
+ <exclude>.repository/</exclude>
+ <exclude>**/*.diff</exclude>
+ <exclude>**/*.patch</exclude>
+ <exclude>**/*.avsc</exclude>
+ <exclude>**/*.avro</exclude>
+ <exclude>**/docs/**</exclude>
+ <exclude>**/test/resources/**</exclude>
+ <exclude>**/.settings/*</exclude>
+ <exclude>**/.classpath</exclude>
+ <exclude>**/.project</exclude>
+ <exclude>**/target/**</exclude>
+ <exclude>**/derby.log</exclude>
+ <exclude>**/metastore_db/</exclude>
+ <exclude>.mvn/**</exclude>
+ <exclude>**/exclude-pmd.properties</exclude>
+ </excludes>
+ <consoleOutput>true</consoleOutput>
+ </configuration>
+ <executions>
+ <execution>
+ <id>verify.rat</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <profiles>
+ <profile>
+ <id>compileThrift</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>compileThrift</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <echo file="target/compile-thrift.sh">
+ LICENSE=src/main/thrift/aslv2
+ THRIFT_DIR=src/main/thrift
+ JAVA_DIR=target/generated-sources/thrift
+ mkdir -p $JAVA_DIR 2> /dev/null
+ JSTATUS=$?
+ if [ $JSTATUS -ne 0 ] ; then
+ echo "Could not create $JAVA_DIR. Will not generate thrift files."
+ exit $JSTATUS
+ fi
+ for THRIFT_FILE in `ls $THRIFT_DIR/*.thrift 2> /dev/null`
+ do
+ thrift --gen java:hashcode -o $JAVA_DIR $THRIFT_FILE
+ done
+ SRC_DIR=$JAVA_DIR/gen-java/org/apache/flume/thrift/
+ DEST_DIR=src/main/java/org/apache/flume/thrift
+ if [ ! -d $DEST_DIR ] ; then
+ mkdir $DEST_DIR 2> /dev/null
+ STATUS=$?
+ if [ $STATUS -ne 0 ] ; then
+ echo "Could not create $DEST_DIR. Will not generate thrift files."
+ exit $STATUS
+ fi
+ fi
+ for JAVA_FILE in `ls $SRC_DIR/*.java 2> /dev/null`
+ do
+ echo $JAVA_FILE
+ cat $LICENSE > $JAVA_FILE.tmp
+ cat $JAVA_FILE >> $JAVA_FILE.tmp
+ mv $JAVA_FILE.tmp $JAVA_FILE
+ cp $JAVA_FILE $DEST_DIR
+ done
+ rm -rf $JAVA_DIR
+ </echo>
+ <exec executable="sh" dir="${basedir}" failonerror="true">
+ <arg line="target/compile-thrift.sh"/>
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/thrift/Status.java b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/Status.java
similarity index 95%
rename from flume-ng-sdk/src/main/java/org/apache/flume/thrift/Status.java
rename to flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/Status.java
index 9118d35ed..6a49f52c6 100644
--- a/flume-ng-sdk/src/main/java/org/apache/flume/thrift/Status.java
+++ b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/Status.java
@@ -22,11 +22,9 @@
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
* @generated
*/
-package org.apache.flume.thrift;
+package org.apache.flume.rpc.thrift;
-import java.util.Map;
-import java.util.HashMap;
import org.apache.thrift.TEnum;
public enum Status implements org.apache.thrift.TEnum {
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/thrift/ThriftFlumeEvent.java b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/ThriftFlumeEvent.java
similarity index 99%
rename from flume-ng-sdk/src/main/java/org/apache/flume/thrift/ThriftFlumeEvent.java
rename to flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/ThriftFlumeEvent.java
index 1eeea84d8..54d2bb754 100644
--- a/flume-ng-sdk/src/main/java/org/apache/flume/thrift/ThriftFlumeEvent.java
+++ b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/ThriftFlumeEvent.java
@@ -22,7 +22,7 @@
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
* @generated
*/
-package org.apache.flume.thrift;
+package org.apache.flume.rpc.thrift;
import org.apache.thrift.scheme.IScheme;
import org.apache.thrift.scheme.SchemeFactory;
@@ -40,11 +40,8 @@ import java.util.ArrayList;
import java.util.Map;
import java.util.HashMap;
import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
import java.util.EnumSet;
import java.util.Collections;
-import java.util.BitSet;
import java.nio.ByteBuffer;
import java.util.Arrays;
import javax.annotation.Generated;
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/thrift/ThriftSourceProtocol.java b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/ThriftSourceProtocol.java
similarity index 99%
rename from flume-ng-sdk/src/main/java/org/apache/flume/thrift/ThriftSourceProtocol.java
rename to flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/ThriftSourceProtocol.java
index 72c9a2c2d..703a5941f 100644
--- a/flume-ng-sdk/src/main/java/org/apache/flume/thrift/ThriftSourceProtocol.java
+++ b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/ThriftSourceProtocol.java
@@ -22,7 +22,7 @@
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
* @generated
*/
-package org.apache.flume.thrift;
+package org.apache.flume.rpc.thrift;
import org.apache.thrift.scheme.IScheme;
import org.apache.thrift.scheme.SchemeFactory;
@@ -30,8 +30,6 @@ import org.apache.thrift.scheme.StandardScheme;
import org.apache.thrift.scheme.TupleScheme;
import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
import org.apache.thrift.TException;
import org.apache.thrift.async.AsyncMethodCallback;
import org.apache.thrift.server.AbstractNonblockingServer.*;
@@ -40,13 +38,9 @@ import java.util.ArrayList;
import java.util.Map;
import java.util.HashMap;
import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
import java.util.EnumSet;
import java.util.Collections;
import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
import javax.annotation.Generated;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1076,7 +1070,7 @@ public class ThriftSourceProtocol {
switch (schemeField.id) {
case 0: // SUCCESS
if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.success = org.apache.flume.thrift.Status.findByValue(iprot.readI32());
+ struct.success = Status.findByValue(iprot.readI32());
struct.setSuccessIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
@@ -1134,7 +1128,7 @@ public class ThriftSourceProtocol {
TTupleProtocol iprot = (TTupleProtocol) prot;
BitSet incoming = iprot.readBitSet(1);
if (incoming.get(0)) {
- struct.success = org.apache.flume.thrift.Status.findByValue(iprot.readI32());
+ struct.success = Status.findByValue(iprot.readI32());
struct.setSuccessIsSet(true);
}
}
@@ -1868,7 +1862,7 @@ public class ThriftSourceProtocol {
switch (schemeField.id) {
case 0: // SUCCESS
if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.success = org.apache.flume.thrift.Status.findByValue(iprot.readI32());
+ struct.success = Status.findByValue(iprot.readI32());
struct.setSuccessIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
@@ -1926,7 +1920,7 @@ public class ThriftSourceProtocol {
TTupleProtocol iprot = (TTupleProtocol) prot;
BitSet incoming = iprot.readBitSet(1);
if (incoming.get(0)) {
- struct.success = org.apache.flume.thrift.Status.findByValue(iprot.readI32());
+ struct.success = Status.findByValue(iprot.readI32());
struct.setSuccessIsSet(true);
}
}
diff --git a/flume-ng-auth/src/main/java/org/apache/flume/api/SecureRpcClientFactory.java b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/api/SecureRpcClientFactory.java
similarity index 90%
rename from flume-ng-auth/src/main/java/org/apache/flume/api/SecureRpcClientFactory.java
rename to flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/api/SecureRpcClientFactory.java
index 35356cdcf..cb1aa7b13 100644
--- a/flume-ng-auth/src/main/java/org/apache/flume/api/SecureRpcClientFactory.java
+++ b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/api/SecureRpcClientFactory.java
@@ -16,7 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.api;
+package org.apache.flume.rpc.thrift.api;
+
+import org.apache.flume.api.RpcClient;
+import org.apache.flume.rpc.thrift.client.ThriftRpcClient;
import java.util.Properties;
diff --git a/flume-ng-auth/src/main/java/org/apache/flume/api/SecureThriftRpcClient.java b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/api/SecureThriftRpcClient.java
similarity index 96%
rename from flume-ng-auth/src/main/java/org/apache/flume/api/SecureThriftRpcClient.java
rename to flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/api/SecureThriftRpcClient.java
index 48c08c282..00bfe2fb7 100644
--- a/flume-ng-auth/src/main/java/org/apache/flume/api/SecureThriftRpcClient.java
+++ b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/api/SecureThriftRpcClient.java
@@ -16,12 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.api;
+package org.apache.flume.rpc.thrift.api;
import org.apache.flume.FlumeException;
import org.apache.flume.auth.FlumeAuthenticationUtil;
import org.apache.flume.auth.FlumeAuthenticator;
import org.apache.flume.auth.PrivilegedExecutor;
+import org.apache.flume.rpc.thrift.client.ThriftRpcClient;
import org.apache.thrift.transport.TSaslClientTransport;
import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransport;
@@ -45,7 +46,7 @@ public class SecureThriftRpcClient extends ThriftRpcClient {
private FlumeAuthenticator privilegedExecutor;
@Override
- protected void configure(Properties properties) throws FlumeException {
+ public void configure(Properties properties) throws FlumeException {
super.configure(properties);
serverPrincipal = properties.getProperty(SERVER_PRINCIPAL);
if (serverPrincipal == null || serverPrincipal.isEmpty()) {
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/api/ThriftRpcClient.java b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/client/ThriftRpcClient.java
similarity index 96%
rename from flume-ng-sdk/src/main/java/org/apache/flume/api/ThriftRpcClient.java
rename to flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/client/ThriftRpcClient.java
index 56053bc8b..ec4d1eabb 100644
--- a/flume-ng-sdk/src/main/java/org/apache/flume/api/ThriftRpcClient.java
+++ b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/client/ThriftRpcClient.java
@@ -16,14 +16,18 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.api;
+package org.apache.flume.rpc.thrift.client;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.FlumeException;
-import org.apache.flume.thrift.Status;
-import org.apache.flume.thrift.ThriftFlumeEvent;
-import org.apache.flume.thrift.ThriftSourceProtocol;
+import org.apache.flume.api.AbstractRpcClient;
+import org.apache.flume.api.HostInfo;
+import org.apache.flume.api.RpcClientConfigurationConstants;
+import org.apache.flume.api.SSLContextAwareAbstractRpcClient;
+import org.apache.flume.rpc.thrift.Status;
+import org.apache.flume.rpc.thrift.ThriftFlumeEvent;
+import org.apache.flume.rpc.thrift.ThriftSourceProtocol;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TCompactProtocol;
import org.apache.thrift.transport.TSocket;
@@ -82,6 +86,10 @@ public class ThriftRpcClient extends SSLContextAwareAbstractRpcClient {
private final Random random = new Random();
private String protocol;
+ public static ThriftRpcClient newInstance() {
+ return new ThriftRpcClient();
+ }
+
public ThriftRpcClient() {
stateLock = new ReentrantLock(true);
connState = State.INIT;
@@ -263,7 +271,7 @@ public class ThriftRpcClient extends SSLContextAwareAbstractRpcClient {
}
@Override
- protected void configure(Properties properties) throws FlumeException {
+ public void configure(Properties properties) throws FlumeException {
if (isActive()) {
throw new FlumeException("Attempting to re-configured an already " +
"configured client!");
@@ -285,7 +293,7 @@ public class ThriftRpcClient extends SSLContextAwareAbstractRpcClient {
+ "choose from. Defaulting to 'compact'.");
protocol = COMPACT_PROTOCOL;
}
- batchSize = parseBatchSize(properties);
+ batchSize = AbstractRpcClient.parseBatchSize(properties);
requestTimeout = Long.parseLong(properties.getProperty(
RpcClientConfigurationConstants.CONFIG_REQUEST_TIMEOUT,
String.valueOf(
diff --git a/flume-ng-core/src/main/java/org/apache/flume/sink/ThriftSink.java b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/sink/ThriftSink.java
similarity index 96%
rename from flume-ng-core/src/main/java/org/apache/flume/sink/ThriftSink.java
rename to flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/sink/ThriftSink.java
index bcab731ac..405fbc213 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/sink/ThriftSink.java
+++ b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/sink/ThriftSink.java
@@ -16,14 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.sink;
-
-import java.util.Properties;
+package org.apache.flume.rpc.thrift.sink;
import org.apache.flume.api.RpcClient;
import org.apache.flume.api.RpcClientConfigurationConstants;
import org.apache.flume.api.RpcClientFactory;
-import org.apache.flume.api.SecureRpcClientFactory;
+import org.apache.flume.rpc.thrift.api.SecureRpcClientFactory;
+import org.apache.flume.sink.AbstractRpcSink;
+
+import java.util.Properties;
/**
* <p>
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/ThriftSource.java b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/source/ThriftSource.java
similarity index 97%
rename from flume-ng-core/src/main/java/org/apache/flume/source/ThriftSource.java
rename to flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/source/ThriftSource.java
index 81c2626f3..b48810265 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/source/ThriftSource.java
+++ b/flume-rpc/flume-rpc-thrift/src/main/java/org/apache/flume/rpc/thrift/source/ThriftSource.java
@@ -16,12 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.source;
+package org.apache.flume.rpc.thrift.source;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.flume.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDrivenSource;
@@ -30,24 +29,26 @@ import org.apache.flume.auth.FlumeAuthenticationUtil;
import org.apache.flume.auth.FlumeAuthenticator;
import org.apache.flume.conf.Configurable;
import org.apache.flume.event.EventBuilder;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.instrumentation.SourceCounter;
-import org.apache.flume.thrift.Status;
-import org.apache.flume.thrift.ThriftSourceProtocol;
-import org.apache.flume.thrift.ThriftFlumeEvent;
+import org.apache.flume.rpc.thrift.Status;
+import org.apache.flume.rpc.thrift.ThriftFlumeEvent;
+import org.apache.flume.rpc.thrift.ThriftSourceProtocol;
+import org.apache.flume.source.SslContextAwareAbstractSource;
import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TCompactProtocol;
import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TCompactProtocol;
import org.apache.thrift.protocol.TProtocolFactory;
import org.apache.thrift.server.TNonblockingServer;
import org.apache.thrift.server.TServer;
import org.apache.thrift.server.TThreadPoolServer;
import org.apache.thrift.transport.TNonblockingServerSocket;
import org.apache.thrift.transport.TNonblockingServerTransport;
+import org.apache.thrift.transport.TSSLTransportFactory;
+import org.apache.thrift.transport.TSaslServerTransport;
import org.apache.thrift.transport.TServerSocket;
import org.apache.thrift.transport.TServerTransport;
-import org.apache.thrift.transport.TSSLTransportFactory;
import org.apache.thrift.transport.TTransportFactory;
-import org.apache.thrift.transport.TSaslServerTransport;
import org.apache.thrift.transport.layered.TFastFramedTransport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,14 +62,14 @@ import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
+import java.security.PrivilegedAction;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.HashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
-import java.security.PrivilegedAction;
public class ThriftSource extends SslContextAwareAbstractSource
implements Configurable, EventDrivenSource {
diff --git a/flume-ng-sdk/src/main/thrift/aslv2 b/flume-rpc/flume-rpc-thrift/src/main/thrift/aslv2
similarity index 100%
rename from flume-ng-sdk/src/main/thrift/aslv2
rename to flume-rpc/flume-rpc-thrift/src/main/thrift/aslv2
diff --git a/flume-ng-sdk/src/main/thrift/flume.thrift b/flume-rpc/flume-rpc-thrift/src/main/thrift/flume.thrift
similarity index 100%
rename from flume-ng-sdk/src/main/thrift/flume.thrift
rename to flume-rpc/flume-rpc-thrift/src/main/thrift/flume.thrift
diff --git a/flume-ng-sdk/src/test/java/org/apache/flume/api/TestThriftRpcClient.java b/flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/TestThriftRpcClient.java
similarity index 96%
rename from flume-ng-sdk/src/test/java/org/apache/flume/api/TestThriftRpcClient.java
rename to flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/TestThriftRpcClient.java
index 8ae86e0f4..62255cbdb 100644
--- a/flume-ng-sdk/src/test/java/org/apache/flume/api/TestThriftRpcClient.java
+++ b/flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/TestThriftRpcClient.java
@@ -16,12 +16,16 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.api;
+package org.apache.flume.rpc.thrift;
import junit.framework.Assert;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
+import org.apache.flume.api.RpcClient;
+import org.apache.flume.api.RpcClientConfigurationConstants;
+import org.apache.flume.api.RpcClientFactory;
import org.apache.flume.event.EventBuilder;
+import org.apache.flume.rpc.thrift.client.ThriftRpcClient;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
diff --git a/flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/ThriftTestingSource.java b/flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/ThriftTestingSource.java
new file mode 100644
index 000000000..6e0df8a32
--- /dev/null
+++ b/flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/ThriftTestingSource.java
@@ -0,0 +1,636 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.rpc.thrift;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.flume.Channel;
+import org.apache.flume.ChannelSelector;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.EventDeliveryException;
+import org.apache.flume.FlumeException;
+import org.apache.flume.Transaction;
+import org.apache.flume.api.RpcClient;
+import org.apache.flume.api.RpcClientConfigurationConstants;
+import org.apache.flume.api.RpcClientFactory;
+import org.apache.flume.channel.ChannelProcessor;
+import org.apache.flume.channel.MemoryChannel;
+import org.apache.flume.channel.ReplicatingChannelSelector;
+import org.apache.flume.conf.Configurables;
+import org.apache.flume.event.EventBuilder;
+import org.apache.flume.exception.ChannelException;
+import org.apache.flume.instrumentation.SourceCounter;
+import org.apache.flume.rpc.thrift.client.ThriftRpcClient;
+import org.apache.flume.rpc.thrift.source.ThriftSource;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.protocol.TProtocolFactory;
+import org.apache.thrift.server.THsHaServer;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.transport.TSSLTransportFactory;
+import org.apache.thrift.transport.TNonblockingServerSocket;
+import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TNonblockingServerTransport;
+import org.apache.thrift.transport.TServerTransport;
+import org.apache.thrift.transport.layered.TFastFramedTransport;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyListOf;
+import static org.mockito.Mockito.doThrow;
+
+public class ThriftTestingSource {
+ public final Queue<Event> flumeEvents = new ConcurrentLinkedQueue<Event>();
+ private final TServer server;
+ public int batchCount = 0;
+ public int individualCount = 0;
+ public int incompleteBatches = 0;
+ private AtomicLong delay = null;
+
+ public void setDelay(AtomicLong delay) {
+ this.delay = delay;
+ }
+
+ private class ThriftOKHandler implements ThriftSourceProtocol.Iface {
+
+ public ThriftOKHandler() {
+
+ }
+
+ @Override
+ public Status append(ThriftFlumeEvent event) throws TException {
+ flumeEvents.add(EventBuilder.withBody(event.getBody(), event.getHeaders()));
+ individualCount++;
+ return Status.OK;
+ }
+
+ @Override
+ public Status appendBatch(List<ThriftFlumeEvent> events) throws TException {
+ batchCount++;
+ if (events.size() < 10) {
+ incompleteBatches++;
+ }
+ for (ThriftFlumeEvent event : events) {
+ flumeEvents.add(EventBuilder.withBody(event.getBody(), event.getHeaders()));
+ }
+ return Status.OK;
+ }
+ }
+
+ private class ThriftFailHandler implements ThriftSourceProtocol.Iface {
+
+ @Override
+ public Status append(ThriftFlumeEvent event) throws TException {
+ return Status.FAILED;
+ }
+
+ @Override
+ public Status appendBatch(List<ThriftFlumeEvent> events) throws
+ TException {
+ return Status.FAILED;
+ }
+ }
+
+ private class ThriftErrorHandler implements ThriftSourceProtocol.Iface {
+
+ @Override
+ public Status append(ThriftFlumeEvent event) throws TException {
+ throw new FlumeException("Forced Error");
+ }
+
+ @Override
+ public Status appendBatch(List<ThriftFlumeEvent> events) throws TException {
+ throw new FlumeException("Forced Error");
+ }
+ }
+
+ private class ThriftSlowHandler extends ThriftOKHandler {
+
+ @Override
+ public Status append(ThriftFlumeEvent event) throws TException {
+ try {
+ TimeUnit.MILLISECONDS.sleep(1550);
+ } catch (InterruptedException e) {
+ throw new FlumeException("Error", e);
+ }
+ return super.append(event);
+ }
+
+ @Override
+ public Status appendBatch(List<ThriftFlumeEvent> events) throws TException {
+ try {
+ TimeUnit.MILLISECONDS.sleep(1550);
+ } catch (InterruptedException e) {
+ throw new FlumeException("Error", e);
+ }
+ return super.appendBatch(events);
+ }
+ }
+
+ private class ThriftTimeoutHandler extends ThriftOKHandler {
+
+ @Override
+ public Status append(ThriftFlumeEvent event) throws TException {
+ try {
+ TimeUnit.MILLISECONDS.sleep(5000);
+ } catch (InterruptedException e) {
+ throw new FlumeException("Error", e);
+ }
+ return super.append(event);
+ }
+
+ @Override
+ public Status appendBatch(List<ThriftFlumeEvent> events) throws TException {
+ try {
+ TimeUnit.MILLISECONDS.sleep(5000);
+ } catch (InterruptedException e) {
+ throw new FlumeException("Error", e);
+ }
+ return super.appendBatch(events);
+ }
+ }
+
+ private class ThriftAlternateHandler extends ThriftOKHandler {
+
+ @Override
+ public Status append(ThriftFlumeEvent event) throws TException {
+ try {
+ if (delay != null) {
+ TimeUnit.MILLISECONDS.sleep(delay.get());
+ }
+ } catch (InterruptedException e) {
+ throw new FlumeException("Error", e);
+ }
+ return super.append(event);
+ }
+
+ @Override
+ public Status appendBatch(List<ThriftFlumeEvent> events) throws TException {
+ try {
+ if (delay != null) {
+ TimeUnit.MILLISECONDS.sleep(delay.get());
+ }
+ } catch (InterruptedException e) {
+ throw new FlumeException("Error", e);
+ }
+ return super.appendBatch(events);
+ }
+ }
+
+ private ThriftSourceProtocol.Iface getHandler(String handlerName) {
+ ThriftSourceProtocol.Iface handler = null;
+ if (handlerName.equals(HandlerType.OK.name())) {
+ handler = new ThriftOKHandler();
+ } else if (handlerName.equals(HandlerType.FAIL.name())) {
+ handler = new ThriftFailHandler();
+ } else if (handlerName.equals(HandlerType.ERROR.name())) {
+ handler = new ThriftErrorHandler();
+ } else if (handlerName.equals(HandlerType.SLOW.name())) {
+ handler = new ThriftSlowHandler();
+ } else if (handlerName.equals(HandlerType.TIMEOUT.name())) {
+ handler = new ThriftTimeoutHandler();
+ } else if (handlerName.equals(HandlerType.ALTERNATE.name())) {
+ handler = new ThriftAlternateHandler();
+ }
+ return handler;
+ }
+
+ public ThriftTestingSource(String handlerName, int port, String protocol) throws Exception {
+ TNonblockingServerTransport serverTransport =
+ new TNonblockingServerSocket(new InetSocketAddress("0.0.0.0", port));
+ ThriftSourceProtocol.Iface handler = getHandler(handlerName);
+
+ TProtocolFactory transportProtocolFactory = null;
+ if (protocol != null && protocol == ThriftRpcClient.BINARY_PROTOCOL) {
+ transportProtocolFactory = new TBinaryProtocol.Factory();
+ } else {
+ transportProtocolFactory = new TCompactProtocol.Factory();
+ }
+ server = new THsHaServer(new THsHaServer.Args(serverTransport).processor(
+ new ThriftSourceProtocol.Processor(handler)).protocolFactory(
+ transportProtocolFactory));
+ Executors.newSingleThreadExecutor().submit(new Runnable() {
+ @Override
+ public void run() {
+ server.serve();
+ }
+ });
+ }
+
+ public ThriftTestingSource(String handlerName, int port,
+ String protocol, String keystore,
+ String keystorePassword, String keyManagerType,
+ String keystoreType) throws Exception {
+ TSSLTransportFactory.TSSLTransportParameters params =
+ new TSSLTransportFactory.TSSLTransportParameters();
+ params.setKeyStore(keystore, keystorePassword, keyManagerType, keystoreType);
+
+ TServerSocket serverTransport = TSSLTransportFactory.getServerSocket(
+ port, 10000, InetAddress.getByName("0.0.0.0"), params);
+
+ ThriftSourceProtocol.Iface handler = getHandler(handlerName);
+
+ Class serverClass = Class.forName("org.apache.thrift" +
+ ".server.TThreadPoolServer");
+ Class argsClass = Class.forName("org.apache.thrift.server" +
+ ".TThreadPoolServer$Args");
+ TServer.AbstractServerArgs args = (TServer.AbstractServerArgs) argsClass
+ .getConstructor(TServerTransport.class)
+ .newInstance(serverTransport);
+ Method m = argsClass.getDeclaredMethod("maxWorkerThreads", int.class);
+ m.invoke(args, Integer.MAX_VALUE);
+ TProtocolFactory transportProtocolFactory = null;
+ if (protocol != null && protocol == ThriftRpcClient.BINARY_PROTOCOL) {
+ transportProtocolFactory = new TBinaryProtocol.Factory();
+ } else {
+ transportProtocolFactory = new TCompactProtocol.Factory();
+ }
+ args.protocolFactory(transportProtocolFactory);
+ args.inputTransportFactory(new TFastFramedTransport.Factory());
+ args.outputTransportFactory(new TFastFramedTransport.Factory());
+ args.processor(new ThriftSourceProtocol.Processor<ThriftSourceProtocol.Iface>(handler));
+ server = (TServer) serverClass.getConstructor(argsClass).newInstance(args);
+ Executors.newSingleThreadExecutor().submit(new Runnable() {
+ @Override
+ public void run() {
+ server.serve();
+ }
+ });
+ }
+
+ public enum HandlerType {
+ OK,
+ FAIL,
+ ERROR,
+ SLOW,
+ TIMEOUT,
+ ALTERNATE;
+ }
+
+ public void stop() {
+ server.stop();
+ }
+
+ public static class TestThriftSource {
+
+ private ThriftSource source;
+ private MemoryChannel channel;
+ private RpcClient client;
+ private final Properties props = new Properties();
+ private int port;
+
+ @Before
+ public void setUp() throws IOException {
+ try (ServerSocket socket = new ServerSocket(0)) {
+ port = socket.getLocalPort();
+ }
+ props.clear();
+ props.setProperty("hosts", "h1");
+ props.setProperty("hosts.h1", "0.0.0.0:" + String.valueOf(port));
+ props.setProperty(RpcClientConfigurationConstants.CONFIG_BATCH_SIZE, "10");
+ props.setProperty(RpcClientConfigurationConstants.CONFIG_REQUEST_TIMEOUT, "2000");
+ channel = new MemoryChannel();
+ source = new ThriftSource();
+ }
+
+ @After
+ public void stop() throws Exception {
+ source.stop();
+ }
+
+ private void configureSource() {
+ List<Channel> channels = new ArrayList<Channel>();
+ channels.add(channel);
+
+ ChannelSelector rcs = new ReplicatingChannelSelector();
+ rcs.setChannels(channels);
+
+ source.setChannelProcessor(new ChannelProcessor(rcs));
+ }
+
+ @Ignore("This test is flakey and causes tests to fail pretty often.")
+ @Test
+ public void testAppendSSLWithComponentKeystore() throws Exception {
+
+ Context context = new Context();
+ channel.configure(context);
+ configureSource();
+
+ context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
+ context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
+ context.put("ssl", "true");
+ context.put("keystore", "src/test/resources/keystorefile.jks");
+ context.put("keystore-password", "password");
+ context.put("keystore-type", "JKS");
+
+ Configurables.configure(source, context);
+
+ doAppendSSL();
+ }
+
+ @Ignore("This test is flakey and causes tests to fail pretty often.")
+ @Test
+ public void testAppendSSLWithGlobalKeystore() throws Exception {
+
+ System.setProperty("javax.net.ssl.keyStore", "src/test/resources/keystorefile.jks");
+ System.setProperty("javax.net.ssl.keyStorePassword", "password");
+ System.setProperty("javax.net.ssl.keyStoreType", "JKS");
+
+ Context context = new Context();
+ channel.configure(context);
+ configureSource();
+
+ context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
+ context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
+ context.put("ssl", "true");
+
+ Configurables.configure(source, context);
+
+ doAppendSSL();
+
+ System.clearProperty("javax.net.ssl.keyStore");
+ System.clearProperty("javax.net.ssl.keyStorePassword");
+ System.clearProperty("javax.net.ssl.keyStoreType");
+ }
+
+ private void doAppendSSL() throws EventDeliveryException {
+ Properties sslprops = (Properties)props.clone();
+ sslprops.put("ssl", "true");
+ sslprops.put("truststore", "src/test/resources/truststorefile.jks");
+ sslprops.put("truststore-password", "password");
+ client = RpcClientFactory.getThriftInstance(sslprops);
+
+ source.start();
+ for (int i = 0; i < 30; i++) {
+ client.append(EventBuilder.withBody(String.valueOf(i).getBytes()));
+ }
+ Transaction transaction = channel.getTransaction();
+ transaction.begin();
+
+ for (int i = 0; i < 30; i++) {
+ Event event = channel.take();
+ Assert.assertNotNull(event);
+ Assert.assertEquals(String.valueOf(i), new String(event.getBody()));
+ }
+ transaction.commit();
+ transaction.close();
+ }
+
+ @Test
+ public void testAppend() throws Exception {
+ client = RpcClientFactory.getThriftInstance(props);
+ Context context = new Context();
+ channel.configure(context);
+ configureSource();
+ context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
+ context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
+ Configurables.configure(source, context);
+ source.start();
+ for (int i = 0; i < 30; i++) {
+ client.append(EventBuilder.withBody(String.valueOf(i).getBytes()));
+ }
+ Transaction transaction = channel.getTransaction();
+ transaction.begin();
+
+ for (int i = 0; i < 30; i++) {
+ Event event = channel.take();
+ Assert.assertNotNull(event);
+ Assert.assertEquals(String.valueOf(i), new String(event.getBody()));
+ }
+ transaction.commit();
+ transaction.close();
+ }
+
+ @Test
+ public void testAppendBatch() throws Exception {
+ client = RpcClientFactory.getThriftInstance(props);
+ Context context = new Context();
+ context.put("capacity", "1000");
+ context.put("transactionCapacity", "1000");
+ channel.configure(context);
+ configureSource();
+ context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
+ context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
+ Configurables.configure(source, context);
+ source.start();
+ for (int i = 0; i < 30; i++) {
+ List<Event> events = Lists.newArrayList();
+ for (int j = 0; j < 10; j++) {
+ Map<String, String> hdrs = Maps.newHashMap();
+ hdrs.put("time", String.valueOf(System.currentTimeMillis()));
+ events.add(EventBuilder.withBody(String.valueOf(i).getBytes(), hdrs));
+ }
+ client.appendBatch(events);
+ }
+ Transaction transaction = channel.getTransaction();
+ transaction.begin();
+ long after = System.currentTimeMillis();
+ List<Integer> events = Lists.newArrayList();
+ for (int i = 0; i < 300; i++) {
+ Event event = channel.take();
+ Assert.assertNotNull(event);
+ Assert.assertTrue(Long.valueOf(event.getHeaders().get("time")) <= after);
+ events.add(Integer.parseInt(new String(event.getBody())));
+ }
+ transaction.commit();
+ transaction.close();
+
+ Collections.sort(events);
+
+ int index = 0;
+ //30 batches of 10
+ for (int i = 0; i < 30; i++) {
+ for (int j = 0; j < 10; j++) {
+ Assert.assertEquals(i, events.get(index++).intValue());
+ }
+ }
+ }
+
+ @Test
+ public void testAppendBigBatch() throws Exception {
+ client = RpcClientFactory.getThriftInstance(props);
+ Context context = new Context();
+ context.put("capacity", "3000");
+ context.put("transactionCapacity", "3000");
+ channel.configure(context);
+ configureSource();
+ context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
+ context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
+ Configurables.configure(source, context);
+ source.start();
+ for (int i = 0; i < 5; i++) {
+ List<Event> events = Lists.newArrayList();
+ for (int j = 0; j < 500; j++) {
+ Map<String, String> hdrs = Maps.newHashMap();
+ hdrs.put("time", String.valueOf(System.currentTimeMillis()));
+ events.add(EventBuilder.withBody(String.valueOf(i).getBytes(), hdrs));
+ }
+ client.appendBatch(events);
+ }
+ Transaction transaction = channel.getTransaction();
+ transaction.begin();
+ long after = System.currentTimeMillis();
+ List<Integer> events = Lists.newArrayList();
+ for (int i = 0; i < 2500; i++) {
+ Event event = channel.take();
+ Assert.assertNotNull(event);
+ Assert.assertTrue(Long.valueOf(event.getHeaders().get("time")) < after);
+ events.add(Integer.parseInt(new String(event.getBody())));
+ }
+ transaction.commit();
+ transaction.close();
+
+ Collections.sort(events);
+
+ int index = 0;
+ //10 batches of 500
+ for (int i = 0; i < 5; i++) {
+ for (int j = 0; j < 500; j++) {
+ Assert.assertEquals(i, events.get(index++).intValue());
+ }
+ }
+ }
+
+ @Test
+ public void testMultipleClients() throws Exception {
+ ExecutorService submitter = Executors.newCachedThreadPool();
+ client = RpcClientFactory.getThriftInstance(props);
+ Context context = new Context();
+ context.put("capacity", "1000");
+ context.put("transactionCapacity", "1000");
+ channel.configure(context);
+ configureSource();
+ context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
+ context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
+ Configurables.configure(source, context);
+ source.start();
+ ExecutorCompletionService<Void> completionService = new ExecutorCompletionService<>(submitter);
+ for (int i = 0; i < 30; i++) {
+ completionService.submit(new SubmitHelper(i), null);
+ }
+ //wait for all threads to be done
+
+
+ for (int i = 0; i < 30; i++) {
+ completionService.take();
+ }
+
+ Transaction transaction = channel.getTransaction();
+ transaction.begin();
+ long after = System.currentTimeMillis();
+ List<Integer> events = Lists.newArrayList();
+ for (int i = 0; i < 300; i++) {
+ Event event = channel.take();
+ Assert.assertNotNull(event);
+ Assert.assertTrue(Long.valueOf(event.getHeaders().get("time")) < after);
+ events.add(Integer.parseInt(new String(event.getBody())));
+ }
+ transaction.commit();
+ transaction.close();
+
+ Collections.sort(events);
+
+ int index = 0;
+ //30 batches of 10
+ for (int i = 0; i < 30; i++) {
+ for (int j = 0; j < 10; j++) {
+ Assert.assertEquals(i, events.get(index++).intValue());
+ }
+ }
+ }
+
+ @Test
+ public void testErrorCounterChannelWriteFail() throws Exception {
+ client = RpcClientFactory.getThriftInstance(props);
+ Context context = new Context();
+ context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
+ context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
+ source.configure(context);
+ ChannelProcessor cp = Mockito.mock(ChannelProcessor.class);
+ doThrow(new ChannelException("dummy")).when(cp).processEvent(any(Event.class));
+ doThrow(new ChannelException("dummy")).when(cp).processEventBatch(anyListOf(Event.class));
+ source.setChannelProcessor(cp);
+ source.start();
+ Event event = EventBuilder.withBody("hello".getBytes());
+ try {
+ client.append(event);
+ } catch (EventDeliveryException e) {
+ //
+ }
+ try {
+ client.appendBatch(Arrays.asList(event));
+ } catch (EventDeliveryException e) {
+ //
+ }
+ SourceCounter sc = (SourceCounter) Whitebox.getInternalState(source, "sourceCounter");
+ Assert.assertEquals(2, sc.getChannelWriteFail());
+ source.stop();
+ }
+
+ private class SubmitHelper implements Runnable {
+ private final int i;
+
+ public SubmitHelper(int i) {
+ this.i = i;
+ }
+
+ @Override
+ public void run() {
+ List<Event> events = Lists.newArrayList();
+ for (int j = 0; j < 10; j++) {
+ Map<String, String> hdrs = Maps.newHashMap();
+ hdrs.put("time", String.valueOf(System.currentTimeMillis()));
+ events.add(EventBuilder.withBody(String.valueOf(i).getBytes(), hdrs));
+ }
+ try {
+ client.appendBatch(events);
+ } catch (EventDeliveryException e) {
+ throw new FlumeException(e);
+ }
+ }
+ }
+ }
+}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/sink/TestThriftSink.java b/flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/sink/TestThriftSink.java
similarity index 98%
rename from flume-ng-core/src/test/java/org/apache/flume/sink/TestThriftSink.java
rename to flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/sink/TestThriftSink.java
index a39bbc822..f84ecf873 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/sink/TestThriftSink.java
+++ b/flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/sink/TestThriftSink.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.sink;
+package org.apache.flume.rpc.thrift.sink;
import com.google.common.base.Charsets;
@@ -25,14 +25,14 @@ import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.Sink;
import org.apache.flume.Transaction;
-import org.apache.flume.api.ThriftRpcClient;
-import org.apache.flume.api.ThriftTestingSource;
import org.apache.flume.channel.MemoryChannel;
import org.apache.flume.conf.Configurables;
import org.apache.flume.event.EventBuilder;
import org.apache.flume.lifecycle.LifecycleController;
import org.apache.flume.lifecycle.LifecycleState;
+import org.apache.flume.rpc.thrift.ThriftTestingSource;
+import org.apache.flume.rpc.thrift.client.ThriftRpcClient;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializerType.java b/flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/source/MockSource.java
similarity index 53%
copy from flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializerType.java
copy to flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/source/MockSource.java
index 86d243fe1..6983c906e 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializerType.java
+++ b/flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/source/MockSource.java
@@ -16,27 +16,48 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.flume.serialization;
+package org.apache.flume.rpc.thrift.source;
-import org.apache.flume.annotations.InterfaceAudience;
-import org.apache.flume.annotations.InterfaceStability;
+import org.apache.flume.Source;
+import org.apache.flume.channel.ChannelProcessor;
+import org.apache.flume.lifecycle.LifecycleState;
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public enum EventSerializerType {
- TEXT(BodyTextEventSerializer.Builder.class),
- HEADER_AND_TEXT(HeaderAndBodyTextEventSerializer.Builder.class),
- AVRO_EVENT(FlumeEventAvroEventSerializer.Builder.class),
- OTHER(null);
+public class MockSource implements Source {
- private final Class<? extends EventSerializer.Builder> builderClass;
+ private String name;
- EventSerializerType(Class<? extends EventSerializer.Builder> builderClass) {
- this.builderClass = builderClass;
+ public MockSource() {
}
- public Class<? extends EventSerializer.Builder> getBuilderClass() {
- return builderClass;
+ @Override
+ public void start() {
}
+ @Override
+ public void stop() {
+ }
+
+ @Override
+ public LifecycleState getLifecycleState() {
+ return null;
+ }
+
+ @Override
+ public void setChannelProcessor(ChannelProcessor cp) {
+ }
+
+ @Override
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public ChannelProcessor getChannelProcessor() {
+ return null;
+ }
}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestDefaultSourceFactory.java b/flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/source/TestDefaultSourceFactory.java
similarity index 68%
copy from flume-ng-core/src/test/java/org/apache/flume/source/TestDefaultSourceFactory.java
copy to flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/source/TestDefaultSourceFactory.java
index 97e024f87..0f2df8395 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestDefaultSourceFactory.java
+++ b/flume-rpc/flume-rpc-thrift/src/test/java/org/apache/flume/rpc/thrift/source/TestDefaultSourceFactory.java
@@ -17,10 +17,18 @@
* under the License.
*/
-package org.apache.flume.source;
+package org.apache.flume.rpc.thrift.source;
import org.apache.flume.Source;
import org.apache.flume.SourceFactory;
+import org.apache.flume.source.DefaultSourceFactory;
+import org.apache.flume.source.ExecSource;
+import org.apache.flume.source.MultiportSyslogTCPSource;
+import org.apache.flume.source.NetcatSource;
+import org.apache.flume.source.NetcatUdpSource;
+import org.apache.flume.source.SequenceGeneratorSource;
+import org.apache.flume.source.SyslogTcpSource;
+import org.apache.flume.source.SyslogUDPSource;
import org.apache.flume.source.http.HTTPSource;
import org.junit.Assert;
import org.junit.Before;
@@ -38,20 +46,20 @@ public class TestDefaultSourceFactory {
@Test
public void testDuplicateCreate() {
- Source avroSource1 = sourceFactory.create("avroSource1", "avro");
- Source avroSource2 = sourceFactory.create("avroSource2", "avro");
+ Source thriftSource1 = sourceFactory.create("thriftSource1", "thrift");
+ Source thriftSource2 = sourceFactory.create("thriftSource2", "thrift");
- Assert.assertNotNull(avroSource1);
- Assert.assertNotNull(avroSource2);
- Assert.assertNotSame(avroSource1, avroSource2);
- Assert.assertTrue(avroSource1 instanceof AvroSource);
- Assert.assertTrue(avroSource2 instanceof AvroSource);
+ Assert.assertNotNull(thriftSource1);
+ Assert.assertNotNull(thriftSource2);
+ Assert.assertNotSame(thriftSource1, thriftSource2);
+ Assert.assertTrue(thriftSource1 instanceof ThriftSource);
+ Assert.assertTrue(thriftSource2 instanceof ThriftSource);
- Source s1 = sourceFactory.create("avroSource1", "avro");
- Source s2 = sourceFactory.create("avroSource2", "avro");
+ Source s1 = sourceFactory.create("thriftSource1", "thrift");
+ Source s2 = sourceFactory.create("thriftSource2", "thrift");
- Assert.assertNotSame(avroSource1, s1);
- Assert.assertNotSame(avroSource2, s2);
+ Assert.assertNotSame(thriftSource1, s1);
+ Assert.assertNotSame(thriftSource2, s2);
}
@@ -68,13 +76,10 @@ public class TestDefaultSourceFactory {
verifySourceCreation("netcat-src", "netcat", NetcatSource.class);
verifySourceCreation("netcat-udp-src", "netcatudp", NetcatUdpSource.class);
verifySourceCreation("exec-src", "exec", ExecSource.class);
- verifySourceCreation("avro-src", "avro", AvroSource.class);
verifySourceCreation("syslogtcp-src", "syslogtcp", SyslogTcpSource.class);
verifySourceCreation("multiport_syslogtcp-src", "multiport_syslogtcp",
MultiportSyslogTCPSource.class);
verifySourceCreation("syslogudp-src", "syslogudp", SyslogUDPSource.class);
- verifySourceCreation("spooldir-src", "spooldir",
- SpoolDirectorySource.class);
verifySourceCreation("http-src", "http", HTTPSource.class);
verifySourceCreation("thrift-src", "thrift", ThriftSource.class);
verifySourceCreation("custom-src", MockSource.class.getCanonicalName(),
diff --git a/flume-rpc/flume-rpc-thrift/src/test/resources/certs/gencerts.sh b/flume-rpc/flume-rpc-thrift/src/test/resources/certs/gencerts.sh
new file mode 100755
index 000000000..0fb0263de
--- /dev/null
+++ b/flume-rpc/flume-rpc-thrift/src/test/resources/certs/gencerts.sh
@@ -0,0 +1,31 @@
+mkdir tmp
+rm ../truststorefile.jks
+rm ../keystorefile.jks
+rm ../server.flume-keystore.p12
+# Create the CA key and certificate
+openssl req -config rootca.conf -new -x509 -nodes -keyout tmp/flume-cacert.key -out tmp/flume-ca.crt -days 10960
+# Create the trust store and import the certificate
+keytool -keystore ../truststorefile.jks -storetype jks -importcert -file 'tmp/flume-ca.crt' -keypass password -storepass password -alias flume-cacert -noprompt
+#Import the root certificate
+keytool -keystore ../keystorefile.jks -alias flume-ca -importcert -file tmp/flume-ca.crt -keypass password -storepass password -noprompt
+# Create the client private key in the client key store
+keytool -genkeypair -keyalg RSA -alias client -keystore ../keystorefile.jks -storepass password -keypass password -validity 10960 -keysize 2048 -dname "CN=client.flume, C=US"
+# Create a signing request for the client #
+keytool -keystore ../keystorefile.jks -alias client -certreq -file tmp/client.csr -keypass password -storepass password
+# Sign the client certificate
+openssl x509 -req -CA 'tmp/flume-ca.crt' -CAkey 'tmp/flume-cacert.key' -in tmp/client.csr -out tmp/client.crt_signed -days 10960 -CAcreateserial -passin pass:password
+# Verify the signed certificate
+openssl verify -CAfile 'tmp/flume-ca.crt' tmp/client.crt_signed
+#Import the client's signed certificate
+keytool -keystore ../keystorefile.jks -alias client -importcert -file tmp/client.crt_signed -keypass password -storepass password -noprompt
+#Verify the keystore
+keytool -list -v -keystore ../keystorefile.jks -storepass password
+# Create the server private key in the server key store
+keytool -genkeypair -keyalg RSA -alias server -keystore ../server.flume-keystore.p12 -storepass password -storetype PKCS12 -keypass password -validity 10960 -keysize 2048 -dname "CN=server.flume, C=US"
+# Create a signing request for the server #
+keytool -keystore ../server.flume-keystore.p12 -alias server -certreq -file tmp/server.csr -keypass password -storepass password
+# Sign the server certificate
+openssl x509 -req -CA 'tmp/flume-ca.crt' -CAkey 'tmp/flume-cacert.key' -in tmp/server.csr -out ../server.flume-crt.pem -days 10960 -CAcreateserial -passin pass:password
+# Extract the private key
+openssl pkcs12 -in ../server.flume-keystore.p12 -passin pass:password -nokeys -out ../server.flume.pem
+rm -rf tmp
diff --git a/flume-rpc/flume-rpc-thrift/src/test/resources/certs/rootca.conf b/flume-rpc/flume-rpc-thrift/src/test/resources/certs/rootca.conf
new file mode 100644
index 000000000..722e9c39b
--- /dev/null
+++ b/flume-rpc/flume-rpc-thrift/src/test/resources/certs/rootca.conf
@@ -0,0 +1,9 @@
+[ req ]
+distinguished_name = CA_DN
+prompt = no
+output_password = password
+default_bits = 2048
+
+[ CA_DN ]
+C = US
+CN = flume-ca
diff --git a/flume-rpc/flume-rpc-thrift/src/test/resources/certs/server.conf b/flume-rpc/flume-rpc-thrift/src/test/resources/certs/server.conf
new file mode 100644
index 000000000..7a9fb5835
--- /dev/null
+++ b/flume-rpc/flume-rpc-thrift/src/test/resources/certs/server.conf
@@ -0,0 +1,9 @@
+[ req ]
+distinguished_name = CA_DN
+prompt = no
+output_password = password
+default_bits = 2048
+
+[ CA_DN ]
+C = US
+CN = server.flume
diff --git a/flume-rpc/flume-rpc-thrift/src/test/resources/keystorefile.jks b/flume-rpc/flume-rpc-thrift/src/test/resources/keystorefile.jks
new file mode 100644
index 000000000..d62e95792
Binary files /dev/null and b/flume-rpc/flume-rpc-thrift/src/test/resources/keystorefile.jks differ
diff --git a/flume-ng-tests/src/test/resources/log4j2.xml b/flume-rpc/flume-rpc-thrift/src/test/resources/log4j2.xml
similarity index 93%
copy from flume-ng-tests/src/test/resources/log4j2.xml
copy to flume-rpc/flume-rpc-thrift/src/test/resources/log4j2.xml
index a6e2e30bd..7caf0d2a7 100644
--- a/flume-ng-tests/src/test/resources/log4j2.xml
+++ b/flume-rpc/flume-rpc-thrift/src/test/resources/log4j2.xml
@@ -23,8 +23,8 @@
</Console>
</Appenders>
<Loggers>
+ <Logger name="org.apache.flume" level="INFO"/>
<Root level="INFO">
- <Logger name="org.apache.flume" level="DEBUG"/>
<AppenderRef ref="Console" />
</Root>
</Loggers>
diff --git a/flume-rpc/flume-rpc-thrift/src/test/resources/truststore.jks b/flume-rpc/flume-rpc-thrift/src/test/resources/truststore.jks
new file mode 100644
index 000000000..8c08349e0
Binary files /dev/null and b/flume-rpc/flume-rpc-thrift/src/test/resources/truststore.jks differ
diff --git a/flume-rpc/flume-rpc-thrift/src/test/resources/truststorefile.jks b/flume-rpc/flume-rpc-thrift/src/test/resources/truststorefile.jks
new file mode 100644
index 000000000..a0c3a49ac
Binary files /dev/null and b/flume-rpc/flume-rpc-thrift/src/test/resources/truststorefile.jks differ
diff --git a/flume-rpc/pom.xml b/flume-rpc/pom.xml
new file mode 100644
index 000000000..f474e11f9
--- /dev/null
+++ b/flume-rpc/pom.xml
@@ -0,0 +1,162 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>flume-parent</artifactId>
+ <groupId>org.apache.flume</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../flume-parent/pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-rpc</artifactId>
+ <name>Flume RPC</name>
+ <version>2.0.0-SNAPSHOT</version>
+ <packaging>pom</packaging>
+
+ <properties>
+ <avro.version>1.11.1</avro.version>
+ <checksum-maven-plugin.version>1.11</checksum-maven-plugin.version>
+ <commons-lang.version>2.6</commons-lang.version>
+ <dropwizard-metrics.version>4.1.18</dropwizard-metrics.version>
+ <findsecbugs-plugin.version>1.12.0</findsecbugs-plugin.version>
+ <flume.version>2.0.0-SNAPSHOT</flume.version>
+ <guava.version>29.0-jre</guava.version>
+ <jackson.version>2.17.0</jackson.version>
+ <mockito.version>1.9.0</mockito.version>
+ <mvn-gpg-plugin.version>1.6</mvn-gpg-plugin.version>
+ <mvn-javadoc-plugin.version>2.9</mvn-javadoc-plugin.version>
+ <mvn-paranamer-plugin.version>2.8.3</mvn-paranamer-plugin.version>
+ <mvn-surefire-plugin.version>3.5.5</mvn-surefire-plugin.version>
+ <netty-all.version>4.1.86.Final</netty-all.version>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <rat.version>0.12</rat.version>
+ <scala.version>2.13</scala.version>
+ <scala-library.version>2.13.9</scala-library.version>
+ <thrift.version>0.14.2</thrift.version>
+ <zlib.version>1.1.3</zlib.version>
+ </properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${dropwizard-metrics.version}</version>
+ </dependency>
+
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>${avro.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-compiler</artifactId>
+ <version>${avro.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-ipc-netty</artifactId>
+ <version>${avro.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>${netty-all.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ <version>${thrift.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jzlib</artifactId>
+ <version>${zlib.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <modules>
+ <module>flume-rpc-avro</module>
+ <module>flume-rpc-thrift</module>
+ </modules>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-maven-plugin</artifactId>
+ <version>${avro.version}</version>
+ </plugin>
+ <plugin>
+ <groupId>com.thoughtworks.paranamer</groupId>
+ <artifactId>paranamer-maven-plugin</artifactId>
+ <version>${mvn-paranamer-plugin.version}</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${mvn-surefire-plugin.version}</version>
+ <configuration>
+ <argLine>-Djava.net.preferIPv4Stack=true</argLine>
+ <!-- <argLine>
+ - -add-opens java.base/java.lang=ALL-UNNAMED
+ - -add-opens java.base/java.util=ALL-UNNAMED
+ -Djava.net.preferIPv4Stack=true</argLine> -->
+ <reuseForks>false</reuseForks>
+ <forkCount>1</forkCount>
+ <!--<rerunFailingTestsCount>10</rerunFailingTestsCount>-->
+ <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
+ <redirectTestOutputToFile>${redirectTestOutput}</redirectTestOutputToFile>
+ <includes>
+ <include>${test.include.pattern}</include>
+ </includes>
+ <excludes>
+ <exclude>**/*$*</exclude>
+ <exclude>${test.exclude.pattern}</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+
+</project>
diff --git a/flume-tools/pom.xml b/flume-tools/pom.xml
index 9bd229924..7d80f5d1c 100644
--- a/flume-tools/pom.xml
+++ b/flume-tools/pom.xml
@@ -25,7 +25,7 @@ limitations under the License.
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../flume-parent/pom.xml</relativePath>
</parent>
diff --git a/pom.xml b/pom.xml
index 771e93225..46331fae8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -27,7 +27,7 @@ limitations under the License.
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.flume</groupId>
<artifactId>flume-project</artifactId>
- <version>1.11.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Flume Project</name>
@@ -37,13 +37,13 @@ limitations under the License.
<module>flume-parent</module>
<module>flume-ng-core</module>
<module>flume-ng-configuration</module>
+ <module>flume-rpc</module>
<module>flume-ng-embedded-agent</module>
<module>flume-ng-sinks</module>
<module>flume-ng-sources</module>
<module>flume-ng-node</module>
<module>flume-ng-dist</module>
<module>flume-ng-channels</module>
- <module>flume-ng-clients</module>
<module>flume-ng-sdk</module>
<module>flume-ng-tests</module>
<module>flume-tools</module>