You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@logging.apache.org by rg...@apache.org on 2026/05/23 03:43:09 UTC
(logging-flume) 01/01: Attempt to create an api module
This is an automated email from the ASF dual-hosted git repository.
rgoers pushed a commit to branch api-attempt
in repository https://gitbox.apache.org/repos/asf/logging-flume.git
commit 92e0727288899819352be071da2e2aa3b8ce2f6a
Author: rgoers <rg...@apache.org>
AuthorDate: Fri May 22 20:42:35 2026 -0700
Attempt to create an api module
---
{flume-ng-configuration => flume-api}/pom.xml | 70 +-
.../src/main/java/org/apache/flume/Channel.java | 0
.../java/org/apache/flume/ChannelException.java | 0
.../main/java/org/apache/flume/ChannelFactory.java | 0
.../org/apache/flume/ChannelFullException.java | 0
.../java/org/apache/flume/ChannelSelector.java | 0
.../src/main/java/org/apache/flume/Context.java | 0
.../main/java/org/apache/flume/CounterGroup.java | 0
.../src/main/java/org/apache/flume/Event.java | 0
.../org/apache/flume/EventDeliveryException.java | 0
.../java/org/apache/flume/EventDrivenSource.java | 0
.../main/java/org/apache/flume/FlumeException.java | 0
.../main/java/org/apache/flume/NamedComponent.java | 0
.../main/java/org/apache/flume/PollableSource.java | 0
.../src/main/java/org/apache/flume/Sink.java | 0
.../main/java/org/apache/flume/SinkFactory.java | 0
.../main/java/org/apache/flume/SinkProcessor.java | 0
.../src/main/java/org/apache/flume/SinkRunner.java | 0
.../src/main/java/org/apache/flume/Source.java | 0
.../main/java/org/apache/flume/SourceFactory.java | 0
.../main/java/org/apache/flume/SourceRunner.java | 0
.../main/java/org/apache/flume/Transaction.java | 0
.../java/org/apache/flume/VersionAnnotation.java | 0
.../org/apache/flume/annotations/Disposable.java | 0
.../flume/annotations/InterfaceAudience.java | 0
.../flume/annotations/InterfaceStability.java | 0
.../org/apache/flume/annotations/Recyclable.java | 0
.../org/apache/flume/channel/AbstractChannel.java | 0
.../flume/channel/AbstractChannelSelector.java | 0
.../flume/channel/BasicChannelSemantics.java | 0
.../flume/channel/BasicTransactionSemantics.java | 0
.../org/apache/flume/conf/BatchSizeSupported.java | 0
.../java/org/apache/flume/conf/Configurable.java | 0
.../apache/flume/conf/ConfigurableComponent.java | 0
.../flume/conf/TransactionCapacitySupported.java | 0
.../java/org/apache/flume/event/EventBuilder.java | 0
.../java/org/apache/flume/event/EventHelper.java | 0
.../java/org/apache/flume/event/JSONEvent.java | 0
.../java/org/apache/flume/event/SimpleEvent.java | 0
.../flume/instrumentation/ChannelCounter.java | 0
.../flume/instrumentation/ChannelCounterMBean.java | 0
.../instrumentation/ChannelProcessorCounter.java | 0
.../flume/instrumentation/MonitorService.java | 0
.../instrumentation/MonitoredCounterGroup.java | 0
.../flume/instrumentation/MonitoringType.java | 0
.../apache/flume/instrumentation/SinkCounter.java | 0
.../flume/instrumentation/SinkCounterMBean.java | 0
.../instrumentation/SinkProcessorCounter.java | 0
.../flume/instrumentation/SourceCounter.java | 0
.../flume/instrumentation/SourceCounterMBean.java | 0
.../org/apache/flume/interceptor/Interceptor.java | 0
.../org/apache/flume/lifecycle/LifecycleAware.java | 0
.../flume/lifecycle/LifecycleController.java | 0
.../apache/flume/lifecycle/LifecycleException.java | 0
.../org/apache/flume/lifecycle/LifecycleState.java | 0
.../flume/lifecycle/LifecycleSupervisor.java | 0
.../flume/serialization/EventDeserializer.java | 0
.../org/apache/flume/serialization/EventSerDe.java | 0
.../flume/serialization/EventSerializer.java | 0
.../flume/serialization/LengthMeasurable.java | 0
.../flume/serialization/PositionTracker.java | 0
.../apache/flume/serialization/RemoteMarkable.java | 0
.../org/apache/flume/serialization/Resettable.java | 0
.../org/apache/flume/serialization/Seekable.java | 0
.../flume/sink/AbstractSingleSinkProcessor.java | 0
.../java/org/apache/flume/sink/AbstractSink.java | 0
.../apache/flume/sink/AbstractSinkProcessor.java | 0
.../apache/flume/sink/AbstractSinkSelector.java | 0
.../main/java/org/apache/flume/sink/SinkGroup.java | 0
.../flume/source/AbstractEventDrivenSource.java | 0
.../flume/source/AbstractPollableSource.java | 0
.../org/apache/flume/source/AbstractSource.java | 0
.../apache/flume/source/BasicSourceSemantics.java | 0
flume-bom/pom.xml | 5 +
.../apache/flume/api/SecureRpcClientFactory.java | 40 -
.../apache/flume/api/SecureThriftRpcClient.java | 124 --
.../TestLoadBalancingLog4jAppender.java | 1 -
.../clients/log4jappender/TestLog4jAppender.java | 1 -
.../log4jappender/TestLog4jAppenderWithAvro.java | 1 -
flume-ng-configuration/pom.xml | 4 +
.../org/apache/flume/conf/ComponentConfig.java | 44 +
.../apache/flume/conf/ComponentConfiguration.java | 31 +-
flume-ng-core/pom.xml | 30 +-
.../src/main/avro/TransferStateFileMeta.avsc | 8 -
.../serialization/AbstractAvroEventSerializer.java | 127 --
.../flume/serialization/AvroEventDeserializer.java | 218 ---
.../AvroEventSerializerConfigurationConstants.java | 43 -
.../FlumeEventAvroEventSerializer.java | 70 -
.../org/apache/flume/sink/AbstractRpcSink.java | 416 -----
.../main/java/org/apache/flume/sink/AvroSink.java | 116 --
.../java/org/apache/flume/sink/ThriftSink.java | 121 --
.../java/org/apache/flume/source/AvroSource.java | 371 ----
.../java/org/apache/flume/source/ThriftSource.java | 413 -----
.../java/org/apache/flume/sink/TestAvroSink.java | 913 ---------
.../java/org/apache/flume/sink/TestThriftSink.java | 366 ----
.../org/apache/flume/source/TestAvroSource.java | 610 ------
.../org/apache/flume/source/TestNetcatSource.java | 2 +-
.../org/apache/flume/source/TestThriftSource.java | 383 ----
.../resources/TestResettableFileInputStream_1.avro | Bin 322 -> 0 bytes
.../TestResettableFileInputStream_1.truncated.avro | Bin 318 -> 0 bytes
flume-ng-core/src/test/resources/syslog_event.avsc | 9 -
flume-ng-sdk/pom.xml | 262 ---
flume-ng-sdk/src/main/avro/flume.avdl | 37 -
.../org/apache/flume/api/AbstractRpcClient.java | 94 -
.../org/apache/flume/api/FailoverRpcClient.java | 291 ---
.../main/java/org/apache/flume/api/HostInfo.java | 99 -
.../apache/flume/api/LoadBalancingRpcClient.java | 293 ---
.../org/apache/flume/api/NettyAvroRpcClient.java | 650 -------
.../main/java/org/apache/flume/api/RpcClient.java | 111 --
.../flume/api/RpcClientConfigurationConstants.java | 161 --
.../org/apache/flume/api/RpcClientFactory.java | 253 ---
.../api/SSLContextAwareAbstractRpcClient.java | 77 -
.../java/org/apache/flume/api/ThriftRpcClient.java | 542 ------
.../main/java/org/apache/flume/thrift/Status.java | 69 -
.../org/apache/flume/thrift/ThriftFlumeEvent.java | 572 ------
.../apache/flume/thrift/ThriftSourceProtocol.java | 1937 --------------------
.../java/org/apache/flume/util/OrderSelector.java | 156 --
.../org/apache/flume/util/RandomOrderSelector.java | 52 -
.../apache/flume/util/RoundRobinOrderSelector.java | 57 -
.../main/java/org/apache/flume/util/SSLUtil.java | 146 --
.../apache/flume/util/SpecificOrderIterator.java | 55 -
flume-ng-sdk/src/main/thrift/aslv2 | 18 -
flume-ng-sdk/src/main/thrift/flume.thrift | 38 -
.../java/org/apache/flume/api/RpcTestUtils.java | 362 ----
.../apache/flume/api/TestFailoverRpcClient.java | 169 --
.../flume/api/TestLoadBalancingRpcClient.java | 687 -------
.../apache/flume/api/TestNettyAvroRpcClient.java | 399 ----
.../org/apache/flume/api/TestRpcClientFactory.java | 154 --
.../org/apache/flume/api/TestThriftRpcClient.java | 208 ---
.../org/apache/flume/api/ThriftTestingSource.java | 282 ---
.../org/apache/flume/event/TestEventBuilder.java | 67 -
.../apache/flume/util/AbstractSSLUtilListTest.java | 44 -
.../org/apache/flume/util/AbstractSSLUtilTest.java | 124 --
.../flume/util/SSLUtilExcludeCipherSuitesTest.java | 49 -
.../flume/util/SSLUtilExcludeProtocolsTest.java | 49 -
.../flume/util/SSLUtilIncludeCipherSuitesTest.java | 49 -
.../flume/util/SSLUtilIncludeProtocolsTest.java | 49 -
.../flume/util/SSLUtilKeystorePasswordTest.java | 49 -
.../apache/flume/util/SSLUtilKeystorePathTest.java | 48 -
.../apache/flume/util/SSLUtilKeystoreTypeTest.java | 48 -
.../util/SSLUtilKeystoreTypeWithDefaultTest.java | 64 -
.../flume/util/SSLUtilTruststorePasswordTest.java | 49 -
.../flume/util/SSLUtilTruststorePathTest.java | 48 -
.../flume/util/SSLUtilTruststoreTypeTest.java | 48 -
.../util/SSLUtilTruststoreTypeWithDefaultTest.java | 64 -
flume-ng-sdk/src/test/resources/log4j2.xml | 31 -
flume-ng-sources/pom.xml | 1 -
flume-parent/pom.xml | 103 +-
pom.xml | 2 +-
149 files changed, 119 insertions(+), 13635 deletions(-)
diff --git a/flume-ng-configuration/pom.xml b/flume-api/pom.xml
similarity index 61%
copy from flume-ng-configuration/pom.xml
copy to flume-api/pom.xml
index edb67e188..202eb5bf4 100644
--- a/flume-ng-configuration/pom.xml
+++ b/flume-api/pom.xml
@@ -6,7 +6,7 @@ The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,23 +17,32 @@ limitations under the License.
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
+
<parent>
<artifactId>flume-parent</artifactId>
<groupId>org.apache.flume</groupId>
<version>2.0.0-SNAPSHOT</version>
- <relativePath>../flume-parent/pom.xml</relativePath>
+ <relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>flume-ng-configuration</artifactId>
- <name>Flume NG Configuration</name>
+
+ <artifactId>flume-api</artifactId>
+ <name>Flume API</name>
+ <description>Flume API</description>
<properties>
- <!-- TODO fix spotbugs violations -->
- <spotbugs.maxAllowedViolations>8</spotbugs.maxAllowedViolations>
- <pmd.maxAllowedViolations>17</pmd.maxAllowedViolations>
- <module.name>org.apache.flume.conf</module.name>
+ <!-- TODO fix spotbugs/pmd violations -->
+ <spotbugs.maxAllowedViolations>69</spotbugs.maxAllowedViolations>
+ <pmd.maxAllowedViolations>170</pmd.maxAllowedViolations>
+ <module.name>org.apache.flume.api</module.name>
</properties>
+
<dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.slf4j</groupId>
@@ -41,37 +50,58 @@ limitations under the License.
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
+ <artifactId>log4j-api</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
+ <artifactId>log4j-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-jul</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-sdk</artifactId>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-config-filter-api</artifactId>
- </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
</dependencies>
+
+ <build>
+<plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-jar</id>
+ <goals>
+ <goal>jar</goal>
+ <goal>test-jar</goal>
+ </goals>
+ <phase>package</phase>
+ </execution>
+ </executions>
+ </plugin>
+</plugins>
+ </build>
+
</project>
diff --git a/flume-ng-core/src/main/java/org/apache/flume/Channel.java b/flume-api/src/main/java/org/apache/flume/Channel.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/Channel.java
rename to flume-api/src/main/java/org/apache/flume/Channel.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/ChannelException.java b/flume-api/src/main/java/org/apache/flume/ChannelException.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/ChannelException.java
rename to flume-api/src/main/java/org/apache/flume/ChannelException.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/ChannelFactory.java b/flume-api/src/main/java/org/apache/flume/ChannelFactory.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/ChannelFactory.java
rename to flume-api/src/main/java/org/apache/flume/ChannelFactory.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/ChannelFullException.java b/flume-api/src/main/java/org/apache/flume/ChannelFullException.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/ChannelFullException.java
rename to flume-api/src/main/java/org/apache/flume/ChannelFullException.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/ChannelSelector.java b/flume-api/src/main/java/org/apache/flume/ChannelSelector.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/ChannelSelector.java
rename to flume-api/src/main/java/org/apache/flume/ChannelSelector.java
diff --git a/flume-ng-configuration/src/main/java/org/apache/flume/Context.java b/flume-api/src/main/java/org/apache/flume/Context.java
similarity index 100%
rename from flume-ng-configuration/src/main/java/org/apache/flume/Context.java
rename to flume-api/src/main/java/org/apache/flume/Context.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/CounterGroup.java b/flume-api/src/main/java/org/apache/flume/CounterGroup.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/CounterGroup.java
rename to flume-api/src/main/java/org/apache/flume/CounterGroup.java
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/Event.java b/flume-api/src/main/java/org/apache/flume/Event.java
similarity index 100%
rename from flume-ng-sdk/src/main/java/org/apache/flume/Event.java
rename to flume-api/src/main/java/org/apache/flume/Event.java
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/EventDeliveryException.java b/flume-api/src/main/java/org/apache/flume/EventDeliveryException.java
similarity index 100%
rename from flume-ng-sdk/src/main/java/org/apache/flume/EventDeliveryException.java
rename to flume-api/src/main/java/org/apache/flume/EventDeliveryException.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/EventDrivenSource.java b/flume-api/src/main/java/org/apache/flume/EventDrivenSource.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/EventDrivenSource.java
rename to flume-api/src/main/java/org/apache/flume/EventDrivenSource.java
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/FlumeException.java b/flume-api/src/main/java/org/apache/flume/FlumeException.java
similarity index 100%
rename from flume-ng-sdk/src/main/java/org/apache/flume/FlumeException.java
rename to flume-api/src/main/java/org/apache/flume/FlumeException.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/NamedComponent.java b/flume-api/src/main/java/org/apache/flume/NamedComponent.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/NamedComponent.java
rename to flume-api/src/main/java/org/apache/flume/NamedComponent.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/PollableSource.java b/flume-api/src/main/java/org/apache/flume/PollableSource.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/PollableSource.java
rename to flume-api/src/main/java/org/apache/flume/PollableSource.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/Sink.java b/flume-api/src/main/java/org/apache/flume/Sink.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/Sink.java
rename to flume-api/src/main/java/org/apache/flume/Sink.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/SinkFactory.java b/flume-api/src/main/java/org/apache/flume/SinkFactory.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/SinkFactory.java
rename to flume-api/src/main/java/org/apache/flume/SinkFactory.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/SinkProcessor.java b/flume-api/src/main/java/org/apache/flume/SinkProcessor.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/SinkProcessor.java
rename to flume-api/src/main/java/org/apache/flume/SinkProcessor.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/SinkRunner.java b/flume-api/src/main/java/org/apache/flume/SinkRunner.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/SinkRunner.java
rename to flume-api/src/main/java/org/apache/flume/SinkRunner.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/Source.java b/flume-api/src/main/java/org/apache/flume/Source.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/Source.java
rename to flume-api/src/main/java/org/apache/flume/Source.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/SourceFactory.java b/flume-api/src/main/java/org/apache/flume/SourceFactory.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/SourceFactory.java
rename to flume-api/src/main/java/org/apache/flume/SourceFactory.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/SourceRunner.java b/flume-api/src/main/java/org/apache/flume/SourceRunner.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/SourceRunner.java
rename to flume-api/src/main/java/org/apache/flume/SourceRunner.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/Transaction.java b/flume-api/src/main/java/org/apache/flume/Transaction.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/Transaction.java
rename to flume-api/src/main/java/org/apache/flume/Transaction.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/VersionAnnotation.java b/flume-api/src/main/java/org/apache/flume/VersionAnnotation.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/VersionAnnotation.java
rename to flume-api/src/main/java/org/apache/flume/VersionAnnotation.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/annotations/Disposable.java b/flume-api/src/main/java/org/apache/flume/annotations/Disposable.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/annotations/Disposable.java
rename to flume-api/src/main/java/org/apache/flume/annotations/Disposable.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/annotations/InterfaceAudience.java b/flume-api/src/main/java/org/apache/flume/annotations/InterfaceAudience.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/annotations/InterfaceAudience.java
rename to flume-api/src/main/java/org/apache/flume/annotations/InterfaceAudience.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/annotations/InterfaceStability.java b/flume-api/src/main/java/org/apache/flume/annotations/InterfaceStability.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/annotations/InterfaceStability.java
rename to flume-api/src/main/java/org/apache/flume/annotations/InterfaceStability.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/annotations/Recyclable.java b/flume-api/src/main/java/org/apache/flume/annotations/Recyclable.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/annotations/Recyclable.java
rename to flume-api/src/main/java/org/apache/flume/annotations/Recyclable.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/channel/AbstractChannel.java b/flume-api/src/main/java/org/apache/flume/channel/AbstractChannel.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/channel/AbstractChannel.java
rename to flume-api/src/main/java/org/apache/flume/channel/AbstractChannel.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/channel/AbstractChannelSelector.java b/flume-api/src/main/java/org/apache/flume/channel/AbstractChannelSelector.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/channel/AbstractChannelSelector.java
rename to flume-api/src/main/java/org/apache/flume/channel/AbstractChannelSelector.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/channel/BasicChannelSemantics.java b/flume-api/src/main/java/org/apache/flume/channel/BasicChannelSemantics.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/channel/BasicChannelSemantics.java
rename to flume-api/src/main/java/org/apache/flume/channel/BasicChannelSemantics.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/channel/BasicTransactionSemantics.java b/flume-api/src/main/java/org/apache/flume/channel/BasicTransactionSemantics.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/channel/BasicTransactionSemantics.java
rename to flume-api/src/main/java/org/apache/flume/channel/BasicTransactionSemantics.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/conf/BatchSizeSupported.java b/flume-api/src/main/java/org/apache/flume/conf/BatchSizeSupported.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/conf/BatchSizeSupported.java
rename to flume-api/src/main/java/org/apache/flume/conf/BatchSizeSupported.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/conf/Configurable.java b/flume-api/src/main/java/org/apache/flume/conf/Configurable.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/conf/Configurable.java
rename to flume-api/src/main/java/org/apache/flume/conf/Configurable.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/conf/ConfigurableComponent.java b/flume-api/src/main/java/org/apache/flume/conf/ConfigurableComponent.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/conf/ConfigurableComponent.java
rename to flume-api/src/main/java/org/apache/flume/conf/ConfigurableComponent.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/conf/TransactionCapacitySupported.java b/flume-api/src/main/java/org/apache/flume/conf/TransactionCapacitySupported.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/conf/TransactionCapacitySupported.java
rename to flume-api/src/main/java/org/apache/flume/conf/TransactionCapacitySupported.java
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/event/EventBuilder.java b/flume-api/src/main/java/org/apache/flume/event/EventBuilder.java
similarity index 100%
rename from flume-ng-sdk/src/main/java/org/apache/flume/event/EventBuilder.java
rename to flume-api/src/main/java/org/apache/flume/event/EventBuilder.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/event/EventHelper.java b/flume-api/src/main/java/org/apache/flume/event/EventHelper.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/event/EventHelper.java
rename to flume-api/src/main/java/org/apache/flume/event/EventHelper.java
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/event/JSONEvent.java b/flume-api/src/main/java/org/apache/flume/event/JSONEvent.java
similarity index 100%
rename from flume-ng-sdk/src/main/java/org/apache/flume/event/JSONEvent.java
rename to flume-api/src/main/java/org/apache/flume/event/JSONEvent.java
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/event/SimpleEvent.java b/flume-api/src/main/java/org/apache/flume/event/SimpleEvent.java
similarity index 100%
rename from flume-ng-sdk/src/main/java/org/apache/flume/event/SimpleEvent.java
rename to flume-api/src/main/java/org/apache/flume/event/SimpleEvent.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/ChannelCounter.java b/flume-api/src/main/java/org/apache/flume/instrumentation/ChannelCounter.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/ChannelCounter.java
rename to flume-api/src/main/java/org/apache/flume/instrumentation/ChannelCounter.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/ChannelCounterMBean.java b/flume-api/src/main/java/org/apache/flume/instrumentation/ChannelCounterMBean.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/ChannelCounterMBean.java
rename to flume-api/src/main/java/org/apache/flume/instrumentation/ChannelCounterMBean.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/ChannelProcessorCounter.java b/flume-api/src/main/java/org/apache/flume/instrumentation/ChannelProcessorCounter.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/ChannelProcessorCounter.java
rename to flume-api/src/main/java/org/apache/flume/instrumentation/ChannelProcessorCounter.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/MonitorService.java b/flume-api/src/main/java/org/apache/flume/instrumentation/MonitorService.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/MonitorService.java
rename to flume-api/src/main/java/org/apache/flume/instrumentation/MonitorService.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/MonitoredCounterGroup.java b/flume-api/src/main/java/org/apache/flume/instrumentation/MonitoredCounterGroup.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/MonitoredCounterGroup.java
rename to flume-api/src/main/java/org/apache/flume/instrumentation/MonitoredCounterGroup.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/MonitoringType.java b/flume-api/src/main/java/org/apache/flume/instrumentation/MonitoringType.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/MonitoringType.java
rename to flume-api/src/main/java/org/apache/flume/instrumentation/MonitoringType.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/SinkCounter.java b/flume-api/src/main/java/org/apache/flume/instrumentation/SinkCounter.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/SinkCounter.java
rename to flume-api/src/main/java/org/apache/flume/instrumentation/SinkCounter.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/SinkCounterMBean.java b/flume-api/src/main/java/org/apache/flume/instrumentation/SinkCounterMBean.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/SinkCounterMBean.java
rename to flume-api/src/main/java/org/apache/flume/instrumentation/SinkCounterMBean.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/SinkProcessorCounter.java b/flume-api/src/main/java/org/apache/flume/instrumentation/SinkProcessorCounter.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/SinkProcessorCounter.java
rename to flume-api/src/main/java/org/apache/flume/instrumentation/SinkProcessorCounter.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/SourceCounter.java b/flume-api/src/main/java/org/apache/flume/instrumentation/SourceCounter.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/SourceCounter.java
rename to flume-api/src/main/java/org/apache/flume/instrumentation/SourceCounter.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/instrumentation/SourceCounterMBean.java b/flume-api/src/main/java/org/apache/flume/instrumentation/SourceCounterMBean.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/instrumentation/SourceCounterMBean.java
rename to flume-api/src/main/java/org/apache/flume/instrumentation/SourceCounterMBean.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/interceptor/Interceptor.java b/flume-api/src/main/java/org/apache/flume/interceptor/Interceptor.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/interceptor/Interceptor.java
rename to flume-api/src/main/java/org/apache/flume/interceptor/Interceptor.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/lifecycle/LifecycleAware.java b/flume-api/src/main/java/org/apache/flume/lifecycle/LifecycleAware.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/lifecycle/LifecycleAware.java
rename to flume-api/src/main/java/org/apache/flume/lifecycle/LifecycleAware.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/lifecycle/LifecycleController.java b/flume-api/src/main/java/org/apache/flume/lifecycle/LifecycleController.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/lifecycle/LifecycleController.java
rename to flume-api/src/main/java/org/apache/flume/lifecycle/LifecycleController.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/lifecycle/LifecycleException.java b/flume-api/src/main/java/org/apache/flume/lifecycle/LifecycleException.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/lifecycle/LifecycleException.java
rename to flume-api/src/main/java/org/apache/flume/lifecycle/LifecycleException.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/lifecycle/LifecycleState.java b/flume-api/src/main/java/org/apache/flume/lifecycle/LifecycleState.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/lifecycle/LifecycleState.java
rename to flume-api/src/main/java/org/apache/flume/lifecycle/LifecycleState.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/lifecycle/LifecycleSupervisor.java b/flume-api/src/main/java/org/apache/flume/lifecycle/LifecycleSupervisor.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/lifecycle/LifecycleSupervisor.java
rename to flume-api/src/main/java/org/apache/flume/lifecycle/LifecycleSupervisor.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializer.java b/flume-api/src/main/java/org/apache/flume/serialization/EventDeserializer.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializer.java
rename to flume-api/src/main/java/org/apache/flume/serialization/EventDeserializer.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerDe.java b/flume-api/src/main/java/org/apache/flume/serialization/EventSerDe.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerDe.java
rename to flume-api/src/main/java/org/apache/flume/serialization/EventSerDe.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializer.java b/flume-api/src/main/java/org/apache/flume/serialization/EventSerializer.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializer.java
rename to flume-api/src/main/java/org/apache/flume/serialization/EventSerializer.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/LengthMeasurable.java b/flume-api/src/main/java/org/apache/flume/serialization/LengthMeasurable.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/LengthMeasurable.java
rename to flume-api/src/main/java/org/apache/flume/serialization/LengthMeasurable.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/PositionTracker.java b/flume-api/src/main/java/org/apache/flume/serialization/PositionTracker.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/PositionTracker.java
rename to flume-api/src/main/java/org/apache/flume/serialization/PositionTracker.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/RemoteMarkable.java b/flume-api/src/main/java/org/apache/flume/serialization/RemoteMarkable.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/RemoteMarkable.java
rename to flume-api/src/main/java/org/apache/flume/serialization/RemoteMarkable.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/Resettable.java b/flume-api/src/main/java/org/apache/flume/serialization/Resettable.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/Resettable.java
rename to flume-api/src/main/java/org/apache/flume/serialization/Resettable.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/Seekable.java b/flume-api/src/main/java/org/apache/flume/serialization/Seekable.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/serialization/Seekable.java
rename to flume-api/src/main/java/org/apache/flume/serialization/Seekable.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractSingleSinkProcessor.java b/flume-api/src/main/java/org/apache/flume/sink/AbstractSingleSinkProcessor.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/sink/AbstractSingleSinkProcessor.java
rename to flume-api/src/main/java/org/apache/flume/sink/AbstractSingleSinkProcessor.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractSink.java b/flume-api/src/main/java/org/apache/flume/sink/AbstractSink.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/sink/AbstractSink.java
rename to flume-api/src/main/java/org/apache/flume/sink/AbstractSink.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractSinkProcessor.java b/flume-api/src/main/java/org/apache/flume/sink/AbstractSinkProcessor.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/sink/AbstractSinkProcessor.java
rename to flume-api/src/main/java/org/apache/flume/sink/AbstractSinkProcessor.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractSinkSelector.java b/flume-api/src/main/java/org/apache/flume/sink/AbstractSinkSelector.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/sink/AbstractSinkSelector.java
rename to flume-api/src/main/java/org/apache/flume/sink/AbstractSinkSelector.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/sink/SinkGroup.java b/flume-api/src/main/java/org/apache/flume/sink/SinkGroup.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/sink/SinkGroup.java
rename to flume-api/src/main/java/org/apache/flume/sink/SinkGroup.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/AbstractEventDrivenSource.java b/flume-api/src/main/java/org/apache/flume/source/AbstractEventDrivenSource.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/source/AbstractEventDrivenSource.java
rename to flume-api/src/main/java/org/apache/flume/source/AbstractEventDrivenSource.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/AbstractPollableSource.java b/flume-api/src/main/java/org/apache/flume/source/AbstractPollableSource.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/source/AbstractPollableSource.java
rename to flume-api/src/main/java/org/apache/flume/source/AbstractPollableSource.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/AbstractSource.java b/flume-api/src/main/java/org/apache/flume/source/AbstractSource.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/source/AbstractSource.java
rename to flume-api/src/main/java/org/apache/flume/source/AbstractSource.java
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/BasicSourceSemantics.java b/flume-api/src/main/java/org/apache/flume/source/BasicSourceSemantics.java
similarity index 100%
rename from flume-ng-core/src/main/java/org/apache/flume/source/BasicSourceSemantics.java
rename to flume-api/src/main/java/org/apache/flume/source/BasicSourceSemantics.java
diff --git a/flume-bom/pom.xml b/flume-bom/pom.xml
index e672ac445..5009165b2 100644
--- a/flume-bom/pom.xml
+++ b/flume-bom/pom.xml
@@ -49,6 +49,11 @@ limitations under the License.
<dependencyManagement>
<dependencies>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-auth</artifactId>
diff --git a/flume-ng-auth/src/main/java/org/apache/flume/api/SecureRpcClientFactory.java b/flume-ng-auth/src/main/java/org/apache/flume/api/SecureRpcClientFactory.java
deleted file mode 100644
index 35356cdcf..000000000
--- a/flume-ng-auth/src/main/java/org/apache/flume/api/SecureRpcClientFactory.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.api;
-
-import java.util.Properties;
-
-/**
- * Factory class to construct Flume {@link RPCClient} implementations.
- */
-public class SecureRpcClientFactory {
-
- /**
- * Return a secure {@linkplain org.apache.flume.api.RpcClient} that uses Thrift for communicating
- * with the next hop.
- * @param props
- * @return - An {@linkplain org.apache.flume.api.RpcClient} which uses thrift configured with the
- * given parameters.
- */
- public static RpcClient getThriftInstance(Properties props) {
- ThriftRpcClient client = new SecureThriftRpcClient();
- client.configure(props);
- return client;
- }
-}
diff --git a/flume-ng-auth/src/main/java/org/apache/flume/api/SecureThriftRpcClient.java b/flume-ng-auth/src/main/java/org/apache/flume/api/SecureThriftRpcClient.java
deleted file mode 100644
index 48c08c282..000000000
--- a/flume-ng-auth/src/main/java/org/apache/flume/api/SecureThriftRpcClient.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.api;
-
-import org.apache.flume.FlumeException;
-import org.apache.flume.auth.FlumeAuthenticationUtil;
-import org.apache.flume.auth.FlumeAuthenticator;
-import org.apache.flume.auth.PrivilegedExecutor;
-import org.apache.thrift.transport.TSaslClientTransport;
-import org.apache.thrift.transport.TSocket;
-import org.apache.thrift.transport.TTransport;
-import org.apache.thrift.transport.TTransportException;
-
-import javax.security.auth.callback.CallbackHandler;
-import javax.security.sasl.Sasl;
-import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-public class SecureThriftRpcClient extends ThriftRpcClient {
-
- private static final String CLIENT_PRINCIPAL = "client-principal";
- private static final String CLIENT_KEYTAB = "client-keytab";
- private static final String SERVER_PRINCIPAL = "server-principal";
-
- private String serverPrincipal;
- private FlumeAuthenticator privilegedExecutor;
-
- @Override
- protected void configure(Properties properties) throws FlumeException {
- super.configure(properties);
- serverPrincipal = properties.getProperty(SERVER_PRINCIPAL);
- if (serverPrincipal == null || serverPrincipal.isEmpty()) {
- throw new IllegalArgumentException("Flume in secure mode, but Flume config doesn't "
- + "specify a server principal to use for Kerberos auth.");
- }
- String clientPrincipal = properties.getProperty(CLIENT_PRINCIPAL);
- String keytab = properties.getProperty(CLIENT_KEYTAB);
- this.privilegedExecutor = FlumeAuthenticationUtil.getAuthenticator(clientPrincipal, keytab);
- if (!privilegedExecutor.isAuthenticated()) {
- throw new FlumeException("Authentication failed in Kerberos mode for " +
- "principal " + clientPrincipal + " keytab " + keytab);
- }
- }
-
- @Override
- protected TTransport getTransport(TSocket tsocket) throws Exception {
- Map<String, String> saslProperties = new HashMap<String, String>();
- saslProperties.put(Sasl.QOP, "auth");
- String[] names;
- try {
- names = FlumeAuthenticationUtil.splitKerberosName(serverPrincipal);
- } catch (IOException e) {
- throw new FlumeException(
- "Error while trying to resolve Principal name - " + serverPrincipal, e);
- }
- return new UgiSaslClientTransport(
- "GSSAPI", null, names[0], names[1], saslProperties, null, tsocket, privilegedExecutor);
- }
-
- /**
- * This transport wraps the Sasl transports to set up the right UGI context for open().
- */
- public static class UgiSaslClientTransport extends TSaslClientTransport {
- PrivilegedExecutor privilegedExecutor;
-
- public UgiSaslClientTransport(String mechanism, String authorizationId,
- String protocol, String serverName, Map<String, String> props,
- CallbackHandler cbh, TTransport transport, PrivilegedExecutor privilegedExecutor)
- throws IOException, TTransportException {
- super(mechanism, authorizationId, protocol, serverName, props, cbh, transport);
- this.privilegedExecutor = privilegedExecutor;
- }
-
- /**
- * Open the SASL transport with using the current UserGroupInformation.
- * This is needed to get the current login context stored
- */
- @Override
- public void open() throws FlumeException {
- try {
- this.privilegedExecutor.execute(
- new PrivilegedExceptionAction<Void>() {
- public Void run() throws FlumeException {
- // this is a workaround to using UgiSaslClientTransport.super.open()
- // which results in IllegalAccessError
- callSuperClassOpen();
- return null;
- }
- });
- } catch (InterruptedException e) {
- throw new FlumeException("Interrupted while opening underlying transport", e);
- } catch (Exception e) {
- throw new FlumeException("Failed to open SASL transport", e);
- }
- }
-
- private void callSuperClassOpen() throws FlumeException {
- try {
- super.open();
- } catch (TTransportException e) {
- throw new FlumeException("Failed to open SASL transport", e);
- }
- }
- }
-}
diff --git a/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLoadBalancingLog4jAppender.java b/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLoadBalancingLog4jAppender.java
index 0546c23c1..cd8b42327 100644
--- a/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLoadBalancingLog4jAppender.java
+++ b/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLoadBalancingLog4jAppender.java
@@ -45,7 +45,6 @@ import org.apache.flume.channel.ChannelProcessor;
import org.apache.flume.channel.MemoryChannel;
import org.apache.flume.channel.ReplicatingChannelSelector;
import org.apache.flume.conf.Configurables;
-import org.apache.flume.source.AvroSource;
import org.apache.flume.source.avro.AvroFlumeEvent;
import org.apache.flume.source.avro.Status;
import org.apache.log4j.Level;
diff --git a/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppender.java b/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppender.java
index 891cb669d..960c73a2f 100644
--- a/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppender.java
+++ b/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppender.java
@@ -43,7 +43,6 @@ import org.apache.flume.channel.ChannelProcessor;
import org.apache.flume.channel.MemoryChannel;
import org.apache.flume.channel.ReplicatingChannelSelector;
import org.apache.flume.conf.Configurables;
-import org.apache.flume.source.AvroSource;
import org.apache.flume.source.avro.AvroFlumeEvent;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
diff --git a/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java b/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java
index 23c2d7e8d..25123aca6 100644
--- a/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java
+++ b/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java
@@ -48,7 +48,6 @@ import org.apache.flume.channel.ChannelProcessor;
import org.apache.flume.channel.MemoryChannel;
import org.apache.flume.channel.ReplicatingChannelSelector;
import org.apache.flume.conf.Configurables;
-import org.apache.flume.source.AvroSource;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
diff --git a/flume-ng-configuration/pom.xml b/flume-ng-configuration/pom.xml
index edb67e188..9dbe83cf9 100644
--- a/flume-ng-configuration/pom.xml
+++ b/flume-ng-configuration/pom.xml
@@ -35,6 +35,10 @@ limitations under the License.
<dependencies>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-api</artifactId>
+ </dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
diff --git a/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfig.java b/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfig.java
new file mode 100644
index 000000000..330379b8b
--- /dev/null
+++ b/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfig.java
@@ -0,0 +1,44 @@
+package org.apache.flume.conf;
+
+import java.util.List;
+
+public interface ComponentConfig {
+ boolean isNotFoundConfigClass();
+
+ void setNotFoundConfigClass();
+
+ List<FlumeConfigurationError> getErrors();
+
+ void configure(Context context) throws ConfigurationException;
+
+ String getType();
+
+ void setType(String type);
+
+ String toString();
+
+ String toString(int indentCount);
+
+ String getComponentName();
+
+ public enum ComponentType {
+ OTHER(null),
+ CONFIG_FILTER("ConfigFilter"),
+ SOURCE("Source"),
+ SINK("Sink"),
+ SINK_PROCESSOR("SinkProcessor"),
+ SINKGROUP("Sinkgroup"),
+ CHANNEL("Channel"),
+ CHANNELSELECTOR("ChannelSelector");
+
+ private final String componentType;
+
+ private ComponentType(String type) {
+ componentType = type;
+ }
+
+ public String getComponentType() {
+ return componentType;
+ }
+ }
+}
diff --git a/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfiguration.java b/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfiguration.java
index b7bab1537..2e31fa8de 100644
--- a/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfiguration.java
+++ b/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfiguration.java
@@ -32,7 +32,7 @@ import org.apache.flume.conf.FlumeConfigurationError.ErrorOrWarning;
*
*/
-public abstract class ComponentConfiguration {
+public abstract class ComponentConfiguration implements ComponentConfig {
protected String componentName;
@@ -41,10 +41,12 @@ public abstract class ComponentConfiguration {
protected List<FlumeConfigurationError> errors;
private boolean notFoundConfigClass;
+ @Override
public boolean isNotFoundConfigClass() {
return notFoundConfigClass;
}
+ @Override
public void setNotFoundConfigClass() {
this.notFoundConfigClass = true;
}
@@ -56,10 +58,12 @@ public abstract class ComponentConfiguration {
configured = false;
}
+ @Override
public List<FlumeConfigurationError> getErrors() {
return errors;
}
+ @Override
public void configure(Context context) throws ConfigurationException {
failIfConfigured();
String confType = context.getString(
@@ -85,18 +89,22 @@ public abstract class ComponentConfiguration {
}
}
+ @Override
public String getType() {
return type;
}
+ @Override
public void setType(String type) {
this.type = type;
}
+ @Override
public String toString() {
return toString(0);
}
+ @Override
public String toString(int indentCount) {
StringBuilder indentSb = new StringBuilder();
@@ -116,6 +124,7 @@ public abstract class ComponentConfiguration {
return sb.toString();
}
+ @Override
public String getComponentName() {
return componentName;
}
@@ -124,24 +133,4 @@ public abstract class ComponentConfiguration {
configured = true;
}
- public enum ComponentType {
- OTHER(null),
- CONFIG_FILTER("ConfigFilter"),
- SOURCE("Source"),
- SINK("Sink"),
- SINK_PROCESSOR("SinkProcessor"),
- SINKGROUP("Sinkgroup"),
- CHANNEL("Channel"),
- CHANNELSELECTOR("ChannelSelector");
-
- private final String componentType;
-
- private ComponentType(String type) {
- componentType = type;
- }
-
- public String getComponentType() {
- return componentType;
- }
- }
}
diff --git a/flume-ng-core/pom.xml b/flume-ng-core/pom.xml
index 4b8efeb9a..5f88a3feb 100644
--- a/flume-ng-core/pom.xml
+++ b/flume-ng-core/pom.xml
@@ -248,17 +248,9 @@ limitations under the License.
<dependencies>
-
<dependency>
<groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-sdk</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-sdk</artifactId>
- <classifier>tests</classifier>
- <scope>test</scope>
+ <artifactId>flume-api</artifactId>
</dependency>
<dependency>
@@ -336,21 +328,6 @@ limitations under the License.
<artifactId>commons-lang</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-ipc-netty</artifactId>
- </dependency>
-
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </dependency>
-
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
@@ -399,11 +376,6 @@ limitations under the License.
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.mina</groupId>
<artifactId>mina-core</artifactId>
diff --git a/flume-ng-core/src/main/avro/TransferStateFileMeta.avsc b/flume-ng-core/src/main/avro/TransferStateFileMeta.avsc
deleted file mode 100644
index f2902319e..000000000
--- a/flume-ng-core/src/main/avro/TransferStateFileMeta.avsc
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "namespace": "org.apache.flume.serialization",
- "type": "record",
- "name": "TransferStateFileMeta",
- "fields": [
- {"name": "offset", "type": "long"}
- ]
-}
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/AbstractAvroEventSerializer.java b/flume-ng-core/src/main/java/org/apache/flume/serialization/AbstractAvroEventSerializer.java
deleted file mode 100644
index 7ae77a57e..000000000
--- a/flume-ng-core/src/main/java/org/apache/flume/serialization/AbstractAvroEventSerializer.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.serialization;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-import org.apache.avro.AvroRuntimeException;
-import org.apache.avro.Schema;
-import org.apache.avro.file.CodecFactory;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.io.DatumWriter;
-import org.apache.avro.reflect.ReflectDatumWriter;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.conf.Configurable;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.*;
-
-/**
- * This is a helper class provided to make it straightforward to serialize
- * Flume {@linkplain Event events} into Avro data.
- * @param <T> Data type that can be written in the Schema given below.
- */
-public abstract class AbstractAvroEventSerializer<T>
- implements EventSerializer, Configurable {
-
- private static final Logger logger =
- LoggerFactory.getLogger(AbstractAvroEventSerializer.class);
-
- private DatumWriter<T> writer = null;
- private DataFileWriter<T> dataFileWriter = null;
-
- /**
- * Returns the stream to serialize data into.
- */
- protected abstract OutputStream getOutputStream();
-
- /**
- * Returns the parsed Avro schema corresponding to the data being written
- * and the parameterized type specified.
- */
- protected abstract Schema getSchema();
-
- /**
- * Simple conversion routine used to convert an Event to a type of your
- * choosing. That type must correspond to the Avro schema given by
- * {@link #getSchema()}.
- */
- protected abstract T convert(Event event);
-
- @Override
- public void configure(Context context) {
-
- int syncIntervalBytes =
- context.getInteger(SYNC_INTERVAL_BYTES, DEFAULT_SYNC_INTERVAL_BYTES);
- String compressionCodec =
- context.getString(COMPRESSION_CODEC, DEFAULT_COMPRESSION_CODEC);
-
- writer = new ReflectDatumWriter<T>(getSchema());
- dataFileWriter = new DataFileWriter<T>(writer);
-
- dataFileWriter.setSyncInterval(syncIntervalBytes);
-
- try {
- CodecFactory codecFactory = CodecFactory.fromString(compressionCodec);
- dataFileWriter.setCodec(codecFactory);
- } catch (AvroRuntimeException e) {
- logger.warn("Unable to instantiate avro codec with name (" +
- compressionCodec + "). Compression disabled. Exception follows.", e);
- }
- }
-
- @Override
- public void afterCreate() throws IOException {
- // write the AVRO container format header
- dataFileWriter.create(getSchema(), getOutputStream());
- }
-
- @Override
- public void afterReopen() throws IOException {
- // impossible to initialize DataFileWriter without writing the schema?
- throw new UnsupportedOperationException("Avro API doesn't support append");
- }
-
- @Override
- public void write(Event event) throws IOException {
- T destType = convert(event);
- dataFileWriter.append(destType);
- }
-
- @Override
- public void flush() throws IOException {
- dataFileWriter.flush();
- }
-
-
- @Override
- public void beforeClose() throws IOException {
- // no-op
- }
-
- @Override
- public boolean supportsReopen() {
- return false;
- }
-
-}
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/AvroEventDeserializer.java b/flume-ng-core/src/main/java/org/apache/flume/serialization/AvroEventDeserializer.java
deleted file mode 100644
index e09c6bba2..000000000
--- a/flume-ng-core/src/main/java/org/apache/flume/serialization/AvroEventDeserializer.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flume.serialization;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-import org.apache.avro.Schema;
-import org.apache.avro.SchemaNormalization;
-import org.apache.avro.file.DataFileConstants;
-import org.apache.avro.file.DataFileReader;
-import org.apache.avro.file.SeekableInput;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.io.BinaryEncoder;
-import org.apache.avro.io.EncoderFactory;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.FlumeException;
-import org.apache.flume.event.EventBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-import java.util.List;
-import java.util.Locale;
-
-/**
- * A deserializer that parses Avro container files, generating one Flume event
- * per record in the Avro file, and storing binary avro-encoded records in
- * the Flume event body.
- */
-public class AvroEventDeserializer implements EventDeserializer {
-
- private static final Logger logger = LoggerFactory.getLogger(AvroEventDeserializer.class);
-
- private final AvroSchemaType schemaType;
- private final ResettableInputStream ris;
-
- private Schema schema;
- private byte[] schemaHash;
- private String schemaHashString;
- private DataFileReader<GenericRecord> fileReader;
- private GenericDatumWriter datumWriter;
- private GenericRecord record;
- private ByteArrayOutputStream out;
- private BinaryEncoder encoder;
-
- @VisibleForTesting
- public static enum AvroSchemaType {
- HASH,
- LITERAL;
- }
-
- public static final String CONFIG_SCHEMA_TYPE_KEY = "schemaType";
- public static final String AVRO_SCHEMA_HEADER_HASH
- = "flume.avro.schema.hash";
- public static final String AVRO_SCHEMA_HEADER_LITERAL
- = "flume.avro.schema.literal";
-
- private AvroEventDeserializer(Context context, ResettableInputStream ris) {
- this.ris = ris;
-
- schemaType = AvroSchemaType.valueOf(
- context.getString(CONFIG_SCHEMA_TYPE_KEY,
- AvroSchemaType.HASH.toString()).toUpperCase(Locale.ENGLISH));
- if (schemaType == AvroSchemaType.LITERAL) {
- logger.warn(CONFIG_SCHEMA_TYPE_KEY + " set to " +
- AvroSchemaType.LITERAL.toString() + ", so storing full Avro " +
- "schema in the header of each event, which may be inefficient. " +
- "Consider using the hash of the schema " +
- "instead of the literal schema.");
- }
- }
-
- private void initialize() throws IOException, NoSuchAlgorithmException {
- SeekableResettableInputBridge in = new SeekableResettableInputBridge(ris);
- long pos = in.tell();
- in.seek(0L);
- fileReader = new DataFileReader<GenericRecord>(in,
- new GenericDatumReader<GenericRecord>());
- fileReader.sync(pos);
-
- schema = fileReader.getSchema();
- datumWriter = new GenericDatumWriter(schema);
- out = new ByteArrayOutputStream();
- encoder = EncoderFactory.get().binaryEncoder(out, encoder);
-
- schemaHash = SchemaNormalization.parsingFingerprint("CRC-64-AVRO", schema);
- schemaHashString = Hex.encodeHexString(schemaHash);
- }
-
- @Override
- public Event readEvent() throws IOException {
- if (fileReader.hasNext()) {
- record = fileReader.next(record);
- out.reset();
- datumWriter.write(record, encoder);
- encoder.flush();
- // annotate header with 64-bit schema CRC hash in hex
- Event event = EventBuilder.withBody(out.toByteArray());
- if (schemaType == AvroSchemaType.HASH) {
- event.getHeaders().put(AVRO_SCHEMA_HEADER_HASH, schemaHashString);
- } else {
- event.getHeaders().put(AVRO_SCHEMA_HEADER_LITERAL, schema.toString());
- }
- return event;
- }
- return null;
- }
-
- @Override
- public List<Event> readEvents(int numEvents) throws IOException {
- List<Event> events = Lists.newArrayList();
- for (int i = 0; i < numEvents && fileReader.hasNext(); i++) {
- Event event = readEvent();
- if (event != null) {
- events.add(event);
- }
- }
- return events;
- }
-
- @Override
- public void mark() throws IOException {
- long pos = fileReader.previousSync() - DataFileConstants.SYNC_SIZE;
- if (pos < 0) pos = 0;
- ((RemoteMarkable) ris).markPosition(pos);
- }
-
- @Override
- public void reset() throws IOException {
- long pos = ((RemoteMarkable) ris).getMarkPosition();
- fileReader.sync(pos);
- }
-
- @Override
- public void close() throws IOException {
- ris.close();
- }
-
- public static class Builder implements EventDeserializer.Builder {
-
- @Override
- public EventDeserializer build(Context context, ResettableInputStream in) {
- if (!(in instanceof RemoteMarkable)) {
- throw new IllegalArgumentException("Cannot use this deserializer " +
- "without a RemoteMarkable input stream");
- }
- AvroEventDeserializer deserializer
- = new AvroEventDeserializer(context, in);
- try {
- deserializer.initialize();
- } catch (Exception e) {
- throw new FlumeException("Cannot instantiate deserializer", e);
- }
- return deserializer;
- }
- }
-
- private static class SeekableResettableInputBridge implements SeekableInput {
- ResettableInputStream ris;
-
- public SeekableResettableInputBridge(ResettableInputStream ris) {
- this.ris = ris;
- }
-
- @Override
- public void seek(long p) throws IOException {
- ris.seek(p);
- }
-
- @Override
- public long tell() throws IOException {
- return ris.tell();
- }
-
- @Override
- public long length() throws IOException {
- if (ris instanceof LengthMeasurable) {
- return ((LengthMeasurable) ris).length();
- } else {
- // FIXME: Avro doesn't seem to complain about this,
- // but probably not a great idea...
- return Long.MAX_VALUE;
- }
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- return ris.read(b, off, len);
- }
-
- @Override
- public void close() throws IOException {
- ris.close();
- }
- }
-
-}
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/AvroEventSerializerConfigurationConstants.java b/flume-ng-core/src/main/java/org/apache/flume/serialization/AvroEventSerializerConfigurationConstants.java
deleted file mode 100644
index 470fcea7b..000000000
--- a/flume-ng-core/src/main/java/org/apache/flume/serialization/AvroEventSerializerConfigurationConstants.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.serialization;
-
-import org.apache.avro.file.CodecFactory;
-
-public class AvroEventSerializerConfigurationConstants {
-
- /**
- * Avro sync interval, in approximate bytes
- */
- public static final String SYNC_INTERVAL_BYTES = "syncIntervalBytes";
- public static final int DEFAULT_SYNC_INTERVAL_BYTES = 2048000; // 2MB
-
- /**
- * Avro compression codec. For supported codecs, see Avro's
- * {@link CodecFactory} class.
- */
- public static final String COMPRESSION_CODEC = "compressionCodec";
- public static final String DEFAULT_COMPRESSION_CODEC = "null"; // no codec
-
- /**
- * Avro static Schema URL
- */
- public static final String STATIC_SCHEMA_URL = "schemaURL";
- public static final String DEFAULT_STATIC_SCHEMA_URL = null;
-}
diff --git a/flume-ng-core/src/main/java/org/apache/flume/serialization/FlumeEventAvroEventSerializer.java b/flume-ng-core/src/main/java/org/apache/flume/serialization/FlumeEventAvroEventSerializer.java
deleted file mode 100644
index 88aab597a..000000000
--- a/flume-ng-core/src/main/java/org/apache/flume/serialization/FlumeEventAvroEventSerializer.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.serialization;
-
-import java.io.OutputStream;
-import org.apache.avro.Schema;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-
-public class FlumeEventAvroEventSerializer extends AbstractAvroEventSerializer<Event> {
-
- private static final Schema SCHEMA = new Schema.Parser().parse(
- "{ \"type\":\"record\", \"name\": \"Event\", \"fields\": [" +
- " {\"name\": \"headers\", \"type\": { \"type\": \"map\", \"values\": \"string\" } }, " +
- " {\"name\": \"body\", \"type\": \"bytes\" } ] }");
-
- private final OutputStream out;
-
- private FlumeEventAvroEventSerializer(OutputStream out) {
- this.out = out;
- }
-
- @Override
- protected Schema getSchema() {
- return SCHEMA;
- }
-
- @Override
- protected OutputStream getOutputStream() {
- return out;
- }
-
- /**
- * A no-op for this simple, special-case implementation
- * @param event
- * @return
- */
- @Override
- protected Event convert(Event event) {
- return event;
- }
-
- public static class Builder implements EventSerializer.Builder {
-
- @Override
- public EventSerializer build(Context context, OutputStream out) {
- FlumeEventAvroEventSerializer writer = new FlumeEventAvroEventSerializer(out);
- writer.configure(context);
- return writer;
- }
-
- }
-
-}
diff --git a/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractRpcSink.java b/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractRpcSink.java
deleted file mode 100644
index 12e2874e0..000000000
--- a/flume-ng-core/src/main/java/org/apache/flume/sink/AbstractRpcSink.java
+++ /dev/null
@@ -1,416 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.FlumeException;
-import org.apache.flume.Transaction;
-import org.apache.flume.api.AbstractRpcClient;
-import org.apache.flume.api.RpcClient;
-import org.apache.flume.api.RpcClientConfigurationConstants;
-import org.apache.flume.conf.BatchSizeSupported;
-import org.apache.flume.conf.Configurable;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * This sink provides the basic RPC functionality for Flume. This sink takes
- * several arguments which are used in RPC.
- * This sink forms one half of Flume's tiered collection support. Events sent to
- * this sink are transported over the network to the hostname / port pair using
- * the RPC implementation encapsulated in {@link RpcClient}.
- * The destination is an instance of Flume's {@link org.apache.flume.source
- * .AvroSource} or {@link org.apache.flume.source.ThriftSource} (based on
- * which implementation of this class is used), which
- * allows Flume agents to forward to other Flume agents, forming a tiered
- * collection infrastructure. Of course, nothing prevents one from using this
- * sink to speak to other custom built infrastructure that implements the same
- * RPC protocol.
- * </p>
- * <p>
- * Events are taken from the configured {@link Channel} in batches of the
- * configured <tt>batch-size</tt>. The batch size has no theoretical limits
- * although all events in the batch <b>must</b> fit in memory. Generally, larger
- * batches are far more efficient, but introduce a slight delay (measured in
- * millis) in delivery. The batch behavior is such that underruns (i.e. batches
- * smaller than the configured batch size) are possible. This is a compromise
- * made to maintain low latency of event delivery. If the channel returns a null
- * event, meaning it is empty, the batch is immediately sent, regardless of
- * size. Batch underruns are tracked in the metrics. Empty batches do not incur
- * an RPC roundtrip.
- * </p>
- * <p>
- * <b>Configuration options</b>
- * </p>
- * <table>
- * <tr>
- * <th>Parameter</th>
- * <th>Description</th>
- * <th>Unit (data type)</th>
- * <th>Default</th>
- * </tr>
- * <tr>
- * <td><tt>hostname</tt></td>
- * <td>The hostname to which events should be sent.</td>
- * <td>Hostname or IP (String)</td>
- * <td>none (required)</td>
- * </tr>
- * <tr>
- * <td><tt>port</tt></td>
- * <td>The port to which events should be sent on <tt>hostname</tt>.</td>
- * <td>TCP port (int)</td>
- * <td>none (required)</td>
- * </tr>
- * <tr>
- * <td><tt>batch-size</tt></td>
- * <td>The maximum number of events to send per RPC.</td>
- * <td>events (int)</td>
- * <td>100</td>
- * </tr>
- * <tr>
- * <td><tt>connect-timeout</tt></td>
- * <td>Maximum time to wait for the first Avro handshake and RPC request</td>
- * <td>milliseconds (long)</td>
- * <td>20000</td>
- * </tr>
- * <tr>
- * <td><tt>request-timeout</tt></td>
- * <td>Maximum time to wait RPC requests after the first</td>
- * <td>milliseconds (long)</td>
- * <td>20000</td>
- * </tr>
- * <tr>
- * <td><tt>compression-type</tt></td>
- * <td>Select compression type. Default is "none" and the only compression type available
- * is "deflate"</td>
- * <td>compression type</td>
- * <td>none</td>
- * </tr>
- * <tr>
- * <td><tt>compression-level</tt></td>
- * <td>In the case compression type is "deflate" this value can be between 0-9.
- * 0 being no compression and 1-9 is compression.
- * The higher the number the better the compression. 6 is the default.</td>
- * <td>compression level</td>
- * <td>6</td>
- * </tr>
- * </table>
- * <p>
- * <b>Metrics</b>
- * </p>
- * <p>
- * TODO
- * </p>
- *
- * <strong>Implementation Notes:</strong> Any implementation of this class
- * must override the {@linkplain #initializeRpcClient(Properties)} method.
- * This method will be called whenever this sink needs to create a new
- * connection to the source.
- */
-public abstract class AbstractRpcSink extends AbstractSink implements Configurable,
- BatchSizeSupported {
-
- private static final Logger logger = LoggerFactory.getLogger(AbstractRpcSink.class);
- private String hostname;
- private Integer port;
- private RpcClient client;
- private Properties clientProps;
- private SinkCounter sinkCounter;
- private int cxnResetInterval;
- private AtomicBoolean resetConnectionFlag;
- private final int DEFAULT_CXN_RESET_INTERVAL = 0;
- private final ScheduledExecutorService cxnResetExecutor =
- Executors.newSingleThreadScheduledExecutor(
- new ThreadFactoryBuilder().setNameFormat("Rpc Sink Reset Thread").build());
-
- // batchSize is used in the clients, here it is only used for config validation
- // before the client is configured
- private int batchSize;
-
-
- @Override
- public void configure(Context context) {
- clientProps = new Properties();
-
- hostname = context.getString("hostname");
- port = context.getInteger("port");
-
- Preconditions.checkState(hostname != null, "No hostname specified");
- Preconditions.checkState(port != null, "No port specified");
-
- clientProps.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS, "h1");
- clientProps.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX +
- "h1", hostname + ":" + port);
-
- for (Entry<String, String> entry: context.getParameters().entrySet()) {
- clientProps.setProperty(entry.getKey(), entry.getValue());
- }
-
- batchSize = AbstractRpcClient.parseBatchSize(clientProps);
-
- if (sinkCounter == null) {
- sinkCounter = new SinkCounter(getName());
- }
- cxnResetInterval = context.getInteger("reset-connection-interval",
- DEFAULT_CXN_RESET_INTERVAL);
- if (cxnResetInterval == DEFAULT_CXN_RESET_INTERVAL) {
- logger.info("Connection reset is set to " + String.valueOf(DEFAULT_CXN_RESET_INTERVAL) +
- ". Will not reset connection to next hop");
- }
- }
-
- /**
- * Returns a new {@linkplain RpcClient} instance configured using the given
- * {@linkplain Properties} object. This method is called whenever a new
- * connection needs to be created to the next hop.
- * @param props
- * @return
- */
- protected abstract RpcClient initializeRpcClient(Properties props);
-
- /**
- * If this function is called successively without calling
- * {@see #destroyConnection()}, only the first call has any effect.
- * @throws org.apache.flume.FlumeException if an RPC client connection could not be opened
- */
- private void createConnection() throws FlumeException {
-
- if (client == null) {
- logger.info("Rpc sink {}: Building RpcClient with hostname: {}, " +
- "port: {}",
- new Object[] { getName(), hostname, port });
- try {
- resetConnectionFlag = new AtomicBoolean(false);
- client = initializeRpcClient(clientProps);
- Preconditions.checkNotNull(client, "Rpc Client could not be " +
- "initialized. " + getName() + " could not be started");
- sinkCounter.incrementConnectionCreatedCount();
- if (cxnResetInterval > 0) {
- cxnResetExecutor.schedule(new Runnable() {
- @Override
- public void run() {
- resetConnectionFlag.set(true);
- }
- }, cxnResetInterval, TimeUnit.SECONDS);
- }
- } catch (Exception ex) {
- sinkCounter.incrementConnectionFailedCount();
- if (ex instanceof FlumeException) {
- throw (FlumeException) ex;
- } else {
- throw new FlumeException(ex);
- }
- }
- logger.debug("Rpc sink {}: Created RpcClient: {}", getName(), client);
- }
-
- }
-
- private void resetConnection() {
- try {
- destroyConnection();
- createConnection();
- } catch (Throwable throwable) {
- // Don't rethrow, else this runnable won't get scheduled again.
- logger.error("Error while trying to expire connection", throwable);
- }
- }
-
- private void destroyConnection() {
- if (client != null) {
- logger.debug("Rpc sink {} closing Rpc client: {}", getName(), client);
- try {
- client.close();
- sinkCounter.incrementConnectionClosedCount();
- } catch (FlumeException e) {
- sinkCounter.incrementConnectionFailedCount();
- logger.error("Rpc sink " + getName() + ": Attempt to close Rpc " +
- "client failed. Exception follows.", e);
- }
- }
-
- client = null;
- }
-
- /**
- * Ensure the connection exists and is active.
- * If the connection is not active, destroy it and recreate it.
- *
- * @throws org.apache.flume.FlumeException If there are errors closing or opening the RPC
- * connection.
- */
- private void verifyConnection() throws FlumeException {
- if (client == null) {
- createConnection();
- } else if (!client.isActive()) {
- destroyConnection();
- createConnection();
- }
- }
-
- /**
- * The start() of RpcSink is more of an optimization that allows connection
- * to be created before the process() loop is started. In case it so happens
- * that the start failed, the process() loop will itself attempt to reconnect
- * as necessary. This is the expected behavior since it is possible that the
- * downstream source becomes unavailable in the middle of the process loop
- * and the sink will have to retry the connection again.
- */
- @Override
- public void start() {
- logger.info("Starting {}...", this);
- sinkCounter.start();
- try {
- createConnection();
- } catch (FlumeException e) {
- logger.warn("Unable to create Rpc client using hostname: " + hostname
- + ", port: " + port, e);
-
- /* Try to prevent leaking resources. */
- destroyConnection();
- }
-
- super.start();
-
- logger.info("Rpc sink {} started.", getName());
- }
-
- @Override
- public void stop() {
- logger.info("Rpc sink {} stopping...", getName());
-
- destroyConnection();
- cxnResetExecutor.shutdown();
- try {
- if (cxnResetExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
- cxnResetExecutor.shutdownNow();
- }
- } catch (Exception ex) {
- logger.error("Interrupted while waiting for connection reset executor to shut down");
- }
- sinkCounter.stop();
- super.stop();
-
- logger.info("Rpc sink {} stopped. Metrics: {}", getName(), sinkCounter);
- }
-
- @Override
- public String toString() {
- return "RpcSink " + getName() + " { host: " + hostname + ", port: " +
- port + " }";
- }
-
- @Override
- public Status process() throws EventDeliveryException {
- Status status = Status.READY;
- Channel channel = getChannel();
- Transaction transaction = channel.getTransaction();
-
- if (resetConnectionFlag.get()) {
- resetConnection();
- // if the time to reset is long and the timeout is short
- // this may cancel the next reset request
- // this should however not be an issue
- resetConnectionFlag.set(false);
- }
-
- try {
- transaction.begin();
-
- verifyConnection();
-
- List<Event> batch = Lists.newLinkedList();
-
- for (int i = 0; i < client.getBatchSize(); i++) {
- Event event = channel.take();
-
- if (event == null) {
- break;
- }
-
- batch.add(event);
- }
-
- int size = batch.size();
- int batchSize = client.getBatchSize();
-
- if (size == 0) {
- sinkCounter.incrementBatchEmptyCount();
- status = Status.BACKOFF;
- } else {
- if (size < batchSize) {
- sinkCounter.incrementBatchUnderflowCount();
- } else {
- sinkCounter.incrementBatchCompleteCount();
- }
- sinkCounter.addToEventDrainAttemptCount(size);
- client.appendBatch(batch);
- }
-
- transaction.commit();
- sinkCounter.addToEventDrainSuccessCount(size);
-
- } catch (Throwable t) {
- transaction.rollback();
- if (t instanceof Error) {
- throw (Error) t;
- } else if (t instanceof ChannelException) {
- logger.error("Rpc Sink " + getName() + ": Unable to get event from" +
- " channel " + channel.getName() + ". Exception follows.", t);
- sinkCounter.incrementChannelReadFail();
- status = Status.BACKOFF;
- } else {
- sinkCounter.incrementEventWriteFail();
- destroyConnection();
- throw new EventDeliveryException("Failed to send events", t);
- }
- } finally {
- transaction.close();
- }
-
- return status;
- }
-
- @VisibleForTesting
- RpcClient getUnderlyingClient() {
- return client;
- }
-
- @Override
- public long getBatchSize() {
- return batchSize;
- }
-}
diff --git a/flume-ng-core/src/main/java/org/apache/flume/sink/AvroSink.java b/flume-ng-core/src/main/java/org/apache/flume/sink/AvroSink.java
deleted file mode 100644
index 06991faf0..000000000
--- a/flume-ng-core/src/main/java/org/apache/flume/sink/AvroSink.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.sink;
-
-import java.util.Properties;
-
-import org.apache.flume.Channel;
-import org.apache.flume.Sink;
-import org.apache.flume.api.RpcClient;
-import org.apache.flume.api.RpcClientFactory;
-import org.apache.flume.source.AvroSource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p>
- * A {@link Sink} implementation that can send events to an RPC server (such as
- * Flume's {@link AvroSource}).
- * </p>
- * <p>
- * This sink forms one half of Flume's tiered collection support. Events sent to
- * this sink are transported over the network to the hostname / port pair using
- * the RPC implementation encapsulated in {@link RpcClient}.
- * The destination is an instance of Flume's {@link AvroSource}, which
- * allows Flume agents to forward to other Flume agents, forming a tiered
- * collection infrastructure. Of course, nothing prevents one from using this
- * sink to speak to other custom built infrastructure that implements the same
- * RPC protocol.
- * </p>
- * <p>
- * Events are taken from the configured {@link Channel} in batches of the
- * configured <tt>batch-size</tt>. The batch size has no theoretical limits
- * although all events in the batch <b>must</b> fit in memory. Generally, larger
- * batches are far more efficient, but introduce a slight delay (measured in
- * millis) in delivery. The batch behavior is such that underruns (i.e. batches
- * smaller than the configured batch size) are possible. This is a compromise
- * made to maintain low latency of event delivery. If the channel returns a null
- * event, meaning it is empty, the batch is immediately sent, regardless of
- * size. Batch underruns are tracked in the metrics. Empty batches do not incur
- * an RPC roundtrip.
- * </p>
- * <p>
- * <b>Configuration options</b>
- * </p>
- * <table>
- * <tr>
- * <th>Parameter</th>
- * <th>Description</th>
- * <th>Unit (data type)</th>
- * <th>Default</th>
- * </tr>
- * <tr>
- * <td><tt>hostname</tt></td>
- * <td>The hostname to which events should be sent.</td>
- * <td>Hostname or IP (String)</td>
- * <td>none (required)</td>
- * </tr>
- * <tr>
- * <td><tt>port</tt></td>
- * <td>The port to which events should be sent on <tt>hostname</tt>.</td>
- * <td>TCP port (int)</td>
- * <td>none (required)</td>
- * </tr>
- * <tr>
- * <td><tt>batch-size</tt></td>
- * <td>The maximum number of events to send per RPC.</td>
- * <td>events (int)</td>
- * <td>100</td>
- * </tr>
- * <tr>
- * <td><tt>connect-timeout</tt></td>
- * <td>Maximum time to wait for the first Avro handshake and RPC request</td>
- * <td>milliseconds (long)</td>
- * <td>20000</td>
- * </tr>
- * <tr>
- * <td><tt>request-timeout</tt></td>
- * <td>Maximum time to wait RPC requests after the first</td>
- * <td>milliseconds (long)</td>
- * <td>20000</td>
- * </tr>
- * </table>
- * <p>
- * <b>Metrics</b>
- * </p>
- * <p>
- * TODO
- * </p>
- */
-public class AvroSink extends AbstractRpcSink {
-
- private static final Logger logger = LoggerFactory.getLogger(AvroSink.class);
-
- @Override
- protected RpcClient initializeRpcClient(Properties props) {
- logger.info("Attempting to create Avro Rpc client.");
- return RpcClientFactory.getInstance(props);
- }
-}
diff --git a/flume-ng-core/src/main/java/org/apache/flume/sink/ThriftSink.java b/flume-ng-core/src/main/java/org/apache/flume/sink/ThriftSink.java
deleted file mode 100644
index bcab731ac..000000000
--- a/flume-ng-core/src/main/java/org/apache/flume/sink/ThriftSink.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink;
-
-import java.util.Properties;
-
-import org.apache.flume.api.RpcClient;
-import org.apache.flume.api.RpcClientConfigurationConstants;
-import org.apache.flume.api.RpcClientFactory;
-import org.apache.flume.api.SecureRpcClientFactory;
-
-/**
- * <p>
- * A {@link org.apache.flume.Sink} implementation that can send events to an RPC server (such as
- * Flume's {@link org.apache.flume.source.ThriftSource}).
- * </p>
- * <p>
- * This sink forms one half of Flume's tiered collection support. Events sent to
- * this sink are transported over the network to the hostname / port pair using
- * the RPC implementation encapsulated in {@link RpcClient}.
- * The destination is an instance of Flume's
- * {@link org.apache.flume.source.ThriftSource}, which
- * allows Flume agents to forward to other Flume agents, forming a tiered
- * collection infrastructure. Of course, nothing prevents one from using this
- * sink to speak to other custom built infrastructure that implements the same
- * RPC protocol.
- * </p>
- * <p>
- * Events are taken from the configured {@link org.apache.flume.Channel} in batches of the
- * configured <tt>batch-size</tt>. The batch size has no theoretical limits
- * although all events in the batch <b>must</b> fit in memory. Generally, larger
- * batches are far more efficient, but introduce a slight delay (measured in
- * millis) in delivery. The batch behavior is such that underruns (i.e. batches
- * smaller than the configured batch size) are possible. This is a compromise
- * made to maintain low latency of event delivery. If the channel returns a null
- * event, meaning it is empty, the batch is immediately sent, regardless of
- * size. Batch underruns are tracked in the metrics. Empty batches do not incur
- * an RPC roundtrip.
- * </p>
- * <p>
- * <b>Configuration options</b>
- * </p>
- * <table>
- * <tr>
- * <th>Parameter</th>
- * <th>Description</th>
- * <th>Unit (data type)</th>
- * <th>Default</th>
- * </tr>
- * <tr>
- * <td><tt>hostname</tt></td>
- * <td>The hostname to which events should be sent.</td>
- * <td>Hostname or IP (String)</td>
- * <td>none (required)</td>
- * </tr>
- * <tr>
- * <td><tt>port</tt></td>
- * <td>The port to which events should be sent on <tt>hostname</tt>.</td>
- * <td>TCP port (int)</td>
- * <td>none (required)</td>
- * </tr>
- * <tr>
- * <td><tt>batch-size</tt></td>
- * <td>The maximum number of events to send per RPC.</td>
- * <td>events (int)</td>
- * <td>100</td>
- * </tr>
- * <tr>
- * <td><tt>connect-timeout</tt></td>
- * <td>Maximum time to wait for the first Avro handshake and RPC request</td>
- * <td>milliseconds (long)</td>
- * <td>20000</td>
- * </tr>
- * <tr>
- * <td><tt>request-timeout</tt></td>
- * <td>Maximum time to wait RPC requests after the first</td>
- * <td>milliseconds (long)</td>
- * <td>20000</td>
- * </tr>
- * </table>
- * <p>
- * <b>Metrics</b>
- * </p>
- * <p>
- * TODO
- * </p>
- */
-public class ThriftSink extends AbstractRpcSink {
- @Override
- protected RpcClient initializeRpcClient(Properties props) {
- // Only one thread is enough, since only one sink thread processes
- // transactions at any given time. Each sink owns its own Rpc client.
- props.setProperty(RpcClientConfigurationConstants.CONFIG_CONNECTION_POOL_SIZE,
- String.valueOf(1));
- boolean enableKerberos = Boolean.parseBoolean(
- props.getProperty(RpcClientConfigurationConstants.KERBEROS_KEY, "false"));
- if (enableKerberos) {
- return SecureRpcClientFactory.getThriftInstance(props);
- } else {
- props.setProperty(RpcClientConfigurationConstants.CONFIG_CLIENT_TYPE,
- RpcClientFactory.ClientType.THRIFT.name());
- return RpcClientFactory.getInstance(props);
- }
- }
-}
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/AvroSource.java b/flume-ng-core/src/main/java/org/apache/flume/source/AvroSource.java
deleted file mode 100644
index 4dd07261b..000000000
--- a/flume-ng-core/src/main/java/org/apache/flume/source/AvroSource.java
+++ /dev/null
@@ -1,371 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.source;
-
-import org.apache.avro.ipc.netty.NettyServer;
-import org.apache.avro.ipc.Responder;
-import org.apache.avro.ipc.Server;
-import org.apache.avro.ipc.specific.SpecificResponder;
-import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDrivenSource;
-import org.apache.flume.FlumeException;
-import org.apache.flume.Source;
-import org.apache.flume.conf.Configurable;
-import org.apache.flume.conf.Configurables;
-import org.apache.flume.conf.LogPrivacyUtil;
-import org.apache.flume.event.EventBuilder;
-import org.apache.flume.instrumentation.SourceCounter;
-import org.apache.flume.netty.filter.PatternRule;
-import org.apache.flume.source.avro.AvroFlumeEvent;
-import org.apache.flume.source.avro.AvroSourceProtocol;
-import org.apache.flume.source.avro.Status;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.SSLEngine;
-
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import io.netty.channel.ChannelPipeline;
-import io.netty.handler.codec.compression.JZlibDecoder;
-import io.netty.handler.codec.compression.JZlibEncoder;
-import io.netty.handler.codec.compression.ZlibEncoder;
-import io.netty.handler.ipfilter.IpFilterRule;
-import io.netty.handler.ipfilter.IpFilterRuleType;
-import io.netty.handler.ipfilter.RuleBasedIpFilter;
-import io.netty.handler.ssl.SslHandler;
-
-/**
- * <p>
- * A {@link Source} implementation that receives Avro events from clients that
- * implement {@link AvroSourceProtocol}.
- * </p>
- * <p>
- * This source forms one half of Flume's tiered collection support. Internally,
- * this source uses Avro's <tt>NettyTransceiver</tt> to listen for, and handle
- * events. It can be paired with the builtin <tt>AvroSink</tt> to create tiered
- * collection topologies. Of course, nothing prevents one from using this source
- * to receive data from other custom built infrastructure that uses the same
- * Avro protocol (specifically {@link AvroSourceProtocol}).
- * </p>
- * <p>
- * Events may be received from the client either singly or in batches.Generally,
- * larger batches are far more efficient, but introduce a slight delay (measured
- * in millis) in delivery. A batch submitted to the configured {@link Channel}
- * atomically (i.e. either all events make it into the channel or none).
- * </p>
- * <p>
- * <b>Configuration options</b>
- * </p>
- * <table>
- * <tr>
- * <th>Parameter</th>
- * <th>Description</th>
- * <th>Unit / Type</th>
- * <th>Default</th>
- * </tr>
- * <tr>
- * <td><tt>bind</tt></td>
- * <td>The hostname or IP to which the source will bind.</td>
- * <td>Hostname or IP / String</td>
- * <td>none (required)</td>
- * </tr>
- * <tr>
- * <td><tt>port</tt></td>
- * <td>The port to which the source will bind and listen for events.</td>
- * <td>TCP port / int</td>
- * <td>none (required)</td>
- * </tr>
- * <tr>
- * <td><tt>threads</tt></td>
- * <td>Max number of threads assigned to thread pool, 0 being unlimited</td>
- * <td>Count / int</td>
- * <td>0(optional)</td>
- * </tr>
- * </table>
- * <p>
- * <b>Metrics</b>
- * </p>
- */
-public class AvroSource extends SslContextAwareAbstractSource implements EventDrivenSource,
- Configurable, AvroSourceProtocol {
-
- private static final String THREADS = "threads";
-
- private static final Logger logger = LoggerFactory.getLogger(AvroSource.class);
-
- private static final String PORT_KEY = "port";
- private static final String BIND_KEY = "bind";
- private static final String COMPRESSION_TYPE = "compression-type";
- private static final String IP_FILTER_KEY = "ipFilter";
- private static final String IP_FILTER_RULES_KEY = "ipFilterRules";
- private int port;
- private String bindAddress;
- private String compressionType;
- private boolean enableIpFilter;
- private String patternRuleConfigDefinition;
-
- private Server server;
- private SourceCounter sourceCounter;
-
- private int maxThreads;
- private ScheduledExecutorService connectionCountUpdater;
-
- private List<IpFilterRule> rules;
-
- @Override
- public void configure(Context context) {
- configureSsl(context);
- Configurables.ensureRequiredNonNull(context, PORT_KEY, BIND_KEY);
-
- port = context.getInteger(PORT_KEY);
- bindAddress = context.getString(BIND_KEY);
- compressionType = context.getString(COMPRESSION_TYPE, "none");
-
- try {
- maxThreads = context.getInteger(THREADS, 0);
- } catch (NumberFormatException e) {
- logger.warn("AVRO source\'s \"threads\" property must specify an integer value.",
- context.getString(THREADS));
- }
-
- enableIpFilter = context.getBoolean(IP_FILTER_KEY, false);
- if (enableIpFilter) {
- patternRuleConfigDefinition = context.getString(IP_FILTER_RULES_KEY);
- if (patternRuleConfigDefinition == null ||
- patternRuleConfigDefinition.trim().isEmpty()) {
- throw new FlumeException(
- "ipFilter is configured with true but ipFilterRules is not defined:" +
- " ");
- }
- String[] patternRuleDefinitions = patternRuleConfigDefinition.split(
- ",");
- rules = new ArrayList<IpFilterRule>(patternRuleDefinitions.length);
- for (String patternRuleDefinition : patternRuleDefinitions) {
- rules.add(generateRule(patternRuleDefinition));
- }
- }
-
- if (sourceCounter == null) {
- sourceCounter = new SourceCounter(getName());
- }
- }
-
- @Override
- public void start() {
- logger.info("Starting {}...", this);
-
- try {
- Responder responder = new SpecificResponder(AvroSourceProtocol.class, this);
- boolean enableCompression = compressionType.equalsIgnoreCase("deflate");
-
- server = new NettyServer(responder, new InetSocketAddress(bindAddress, port),
- (ch) -> {
- ChannelPipeline pipeline = ch.pipeline();
- if (enableCompression) {
- ZlibEncoder encoder = new JZlibEncoder(6);
- pipeline.addFirst("deflater", encoder);
- pipeline.addFirst("inflater", new JZlibDecoder());
- }
- Optional<SSLEngine> engine = getSslEngine(false);
- engine.ifPresent(sslEngine -> pipeline.addLast("ssl", new SslHandler(sslEngine)));
- if (enableIpFilter) {
- logger.info("Setting up ipFilter with the following rule definition: " +
- patternRuleConfigDefinition);
- RuleBasedIpFilter filter = new RuleBasedIpFilter(rules.toArray(new IpFilterRule[0]));
- logger.info("Adding ipFilter with " + rules.size() + " rules");
- pipeline.addFirst("ipFilter", filter);
- }
- });
- } catch (Exception nce) {
- logger.error("Avro source {} startup failed. Cannot initialize Netty server", getName(), nce);
- stop();
- throw new FlumeException("Failed to set up server socket", nce);
- }
-
- connectionCountUpdater = Executors.newSingleThreadScheduledExecutor();
- server.start();
- sourceCounter.start();
- super.start();
- final NettyServer srv = (NettyServer)server;
- connectionCountUpdater.scheduleWithFixedDelay(
- () -> sourceCounter.setOpenConnectionCount(Long.valueOf(srv.getNumActiveConnections())),
- 0, 60, TimeUnit.SECONDS);
-
- logger.info("Avro source {} started.", getName());
- }
-
- @Override
- public void stop() {
- logger.info("Avro source {} stopping: {}", getName(), this);
-
- if (server != null) {
- server.close();
- try {
- server.join();
- server = null;
- } catch (InterruptedException e) {
- logger.info("Avro source " + getName() + ": Interrupted while waiting " +
- "for Avro server to stop. Exiting. Exception follows.", e);
- Thread.currentThread().interrupt();
- }
- }
-
- sourceCounter.stop();
- if (connectionCountUpdater != null) {
- connectionCountUpdater.shutdownNow();
- connectionCountUpdater = null;
- }
-
- super.stop();
- logger.info("Avro source {} stopped. Metrics: {}", getName(), sourceCounter);
- }
-
- @Override
- public String toString() {
- return "Avro source " + getName() + ": { bindAddress: " + bindAddress +
- ", port: " + port + " }";
- }
-
- /**
- * Helper function to convert a map of CharSequence to a map of String.
- */
- private static Map<String, String> toStringMap(Map<CharSequence, CharSequence> charSeqMap) {
- Map<String, String> stringMap = new HashMap<>();
- for (Map.Entry<CharSequence, CharSequence> entry : charSeqMap.entrySet()) {
- stringMap.put(entry.getKey().toString(), entry.getValue().toString());
- }
- return stringMap;
- }
-
- @Override
- public Status append(AvroFlumeEvent avroEvent) {
- if (logger.isDebugEnabled()) {
- if (LogPrivacyUtil.allowLogRawData()) {
- logger.debug("Avro source {}: Received avro event: {}", getName(), avroEvent);
- } else {
- logger.debug("Avro source {}: Received avro event", getName());
- }
- }
-
- sourceCounter.incrementAppendReceivedCount();
- sourceCounter.incrementEventReceivedCount();
-
- Event event = EventBuilder.withBody(avroEvent.getBody().array(),
- toStringMap(avroEvent.getHeaders()));
-
- try {
- getChannelProcessor().processEvent(event);
- } catch (ChannelException ex) {
- logger.warn("Avro source " + getName() + ": Unable to process event. " +
- "Exception follows.", ex);
- sourceCounter.incrementChannelWriteFail();
- return Status.FAILED;
- }
-
- sourceCounter.incrementAppendAcceptedCount();
- sourceCounter.incrementEventAcceptedCount();
-
- return Status.OK;
- }
-
- @Override
- public Status appendBatch(List<AvroFlumeEvent> events) {
- logger.debug("Avro source {}: Received avro event batch of {} events.",
- getName(), events.size());
- sourceCounter.incrementAppendBatchReceivedCount();
- sourceCounter.addToEventReceivedCount(events.size());
-
- List<Event> batch = new ArrayList<Event>();
-
- for (AvroFlumeEvent avroEvent : events) {
- Event event = EventBuilder.withBody(avroEvent.getBody().array(),
- toStringMap(avroEvent.getHeaders()));
-
- batch.add(event);
- }
-
- try {
- getChannelProcessor().processEventBatch(batch);
- } catch (Throwable t) {
- logger.error("Avro source " + getName() + ": Unable to process event " +
- "batch. Exception follows.", t);
- sourceCounter.incrementChannelWriteFail();
- if (t instanceof Error) {
- throw (Error) t;
- }
- return Status.FAILED;
- }
-
- sourceCounter.incrementAppendBatchAcceptedCount();
- sourceCounter.addToEventAcceptedCount(events.size());
-
- return Status.OK;
- }
-
- private PatternRule generateRule(String patternRuleDefinition) throws FlumeException {
- patternRuleDefinition = patternRuleDefinition.trim();
- //first validate the format
- int firstColonIndex = patternRuleDefinition.indexOf(":");
- if (firstColonIndex == -1) {
- throw new FlumeException(
- "Invalid ipFilter patternRule '" + patternRuleDefinition +
- "' should look like <'allow' or 'deny'>:<'ip' or " +
- "'name'>:<pattern>");
- } else {
- String ruleAccessFlag = patternRuleDefinition.substring(0, firstColonIndex);
- int secondColonIndex = patternRuleDefinition.indexOf(":", firstColonIndex + 1);
- if ((!ruleAccessFlag.equals("allow") && !ruleAccessFlag.equals("deny")) ||
- secondColonIndex == -1) {
- throw new FlumeException(
- "Invalid ipFilter patternRule '" + patternRuleDefinition +
- "' should look like <'allow' or 'deny'>:<'ip' or " +
- "'name'>:<pattern>");
- }
-
- String patternTypeFlag = patternRuleDefinition.substring(
- firstColonIndex + 1, secondColonIndex);
- if ((!patternTypeFlag.equals("ip") && !patternTypeFlag.equals("name"))) {
- throw new FlumeException(
- "Invalid ipFilter patternRule '" + patternRuleDefinition +
- "' should look like <'allow' or 'deny'>:<'ip' or " +
- "'name'>:<pattern>");
- }
-
- boolean isAllow = ruleAccessFlag.equals("allow");
- String patternRuleString = (patternTypeFlag.equals("ip") ? "i" : "n")
- + ":" + patternRuleDefinition.substring(secondColonIndex + 1);
- IpFilterRuleType ruleType = isAllow ? IpFilterRuleType.ACCEPT : IpFilterRuleType.REJECT;
- logger.info("Adding ipFilter PatternRule: {} {}", ruleType, patternRuleString);
- return new PatternRule(ruleType, patternRuleString);
- }
- }
-}
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/ThriftSource.java b/flume-ng-core/src/main/java/org/apache/flume/source/ThriftSource.java
deleted file mode 100644
index 81c2626f3..000000000
--- a/flume-ng-core/src/main/java/org/apache/flume/source/ThriftSource.java
+++ /dev/null
@@ -1,413 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.source;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.flume.ChannelException;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDrivenSource;
-import org.apache.flume.FlumeException;
-import org.apache.flume.auth.FlumeAuthenticationUtil;
-import org.apache.flume.auth.FlumeAuthenticator;
-import org.apache.flume.conf.Configurable;
-import org.apache.flume.event.EventBuilder;
-import org.apache.flume.instrumentation.SourceCounter;
-import org.apache.flume.thrift.Status;
-import org.apache.flume.thrift.ThriftSourceProtocol;
-import org.apache.flume.thrift.ThriftFlumeEvent;
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TCompactProtocol;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.protocol.TProtocolFactory;
-import org.apache.thrift.server.TNonblockingServer;
-import org.apache.thrift.server.TServer;
-import org.apache.thrift.server.TThreadPoolServer;
-import org.apache.thrift.transport.TNonblockingServerSocket;
-import org.apache.thrift.transport.TNonblockingServerTransport;
-import org.apache.thrift.transport.TServerSocket;
-import org.apache.thrift.transport.TServerTransport;
-import org.apache.thrift.transport.TSSLTransportFactory;
-import org.apache.thrift.transport.TTransportFactory;
-import org.apache.thrift.transport.TSaslServerTransport;
-import org.apache.thrift.transport.layered.TFastFramedTransport;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLParameters;
-import javax.net.ssl.SSLServerSocket;
-import javax.security.sasl.Sasl;
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.util.List;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.security.PrivilegedAction;
-
-public class ThriftSource extends SslContextAwareAbstractSource
- implements Configurable, EventDrivenSource {
-
- public static final Logger logger = LoggerFactory.getLogger(ThriftSource.class);
-
- /**
- * Config param for the maximum number of threads this source should use to
- * handle incoming data.
- */
- public static final String CONFIG_THREADS = "threads";
- /**
- * Config param for the hostname to listen on.
- */
- public static final String CONFIG_BIND = "bind";
- /**
- * Config param for the port to listen on.
- */
- public static final String CONFIG_PORT = "port";
- /**
- * Config param for the thrift protocol to use.
- */
- public static final String CONFIG_PROTOCOL = "protocol";
- public static final String BINARY_PROTOCOL = "binary";
- public static final String COMPACT_PROTOCOL = "compact";
-
- private static final String KERBEROS_KEY = "kerberos";
- private static final String AGENT_PRINCIPAL = "agent-principal";
- private static final String AGENT_KEYTAB = "agent-keytab";
-
- private Integer port;
- private String bindAddress;
- private int maxThreads = 0;
- private SourceCounter sourceCounter;
- private TServer server;
- private ExecutorService servingExecutor;
- private String protocol;
- private boolean enableKerberos = false;
- private String principal;
- private FlumeAuthenticator flumeAuth;
-
- @Override
- public void configure(Context context) {
- configureSsl(context);
- logger.info("Configuring thrift source.");
- port = context.getInteger(CONFIG_PORT);
- Preconditions.checkNotNull(port, "Port must be specified for Thrift " +
- "Source.");
- bindAddress = context.getString(CONFIG_BIND);
- Preconditions.checkNotNull(bindAddress, "Bind address must be specified " +
- "for Thrift Source.");
-
- try {
- maxThreads = context.getInteger(CONFIG_THREADS, 0);
- maxThreads = (maxThreads <= 0) ? Integer.MAX_VALUE : maxThreads;
- } catch (NumberFormatException e) {
- logger.warn("Thrift source\'s \"threads\" property must specify an " +
- "integer value: " + context.getString(CONFIG_THREADS));
- }
-
- if (sourceCounter == null) {
- sourceCounter = new SourceCounter(getName());
- }
-
- protocol = context.getString(CONFIG_PROTOCOL);
- if (protocol == null) {
- // default is to use the compact protocol.
- protocol = COMPACT_PROTOCOL;
- }
- Preconditions.checkArgument(
- (protocol.equalsIgnoreCase(BINARY_PROTOCOL) ||
- protocol.equalsIgnoreCase(COMPACT_PROTOCOL)),
- "binary or compact are the only valid Thrift protocol types to " +
- "choose from.");
-
- principal = context.getString(AGENT_PRINCIPAL);
- String keytab = context.getString(AGENT_KEYTAB);
- enableKerberos = context.getBoolean(KERBEROS_KEY, false);
- this.flumeAuth = FlumeAuthenticationUtil.getAuthenticator(principal, keytab);
- if (enableKerberos) {
- if (!flumeAuth.isAuthenticated()) {
- throw new FlumeException("Authentication failed in Kerberos mode for " +
- "principal " + principal + " keytab " + keytab);
- }
- flumeAuth.startCredentialRefresher();
- }
- }
-
- @Override
- public void start() {
- logger.info("Starting thrift source");
-
- // create the server
- server = getTThreadedSelectorServer();
-
- // if in ssl mode or if SelectorServer is unavailable
- if (server == null) {
- server = getTThreadPoolServer();
- }
-
- servingExecutor = Executors.newSingleThreadExecutor(new
- ThreadFactoryBuilder().setNameFormat("Flume Thrift Source I/O Boss")
- .build());
-
- /**
- * Start serving.
- */
- servingExecutor.submit(new Runnable() {
- @Override
- public void run() {
- flumeAuth.execute(new PrivilegedAction<Object>() {
- @Override
- public Object run() {
- server.serve();
- return null;
- }
- });
- }
- });
-
- long timeAfterStart = System.currentTimeMillis();
- while (!server.isServing()) {
- try {
- if (System.currentTimeMillis() - timeAfterStart >= 10000) {
- throw new FlumeException("Thrift server failed to start!");
- }
- TimeUnit.MILLISECONDS.sleep(1000);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new FlumeException("Interrupted while waiting for Thrift server" +
- " to start.", e);
- }
- }
- sourceCounter.start();
- logger.info("Started Thrift source.");
- super.start();
- }
-
- private TServerTransport getSSLServerTransport() {
- try {
- TServerTransport transport;
- TSSLTransportFactory.TSSLTransportParameters params =
- new TSSLTransportFactory.TSSLTransportParameters();
-
- params.setKeyStore(getKeystore(), getKeystorePassword(),
- KeyManagerFactory.getDefaultAlgorithm(), getKeystoreType());
- transport = TSSLTransportFactory.getServerSocket(
- port, 120000, InetAddress.getByName(bindAddress), params);
-
- ServerSocket serverSock = ((TServerSocket) transport).getServerSocket();
- if (serverSock instanceof SSLServerSocket) {
- SSLServerSocket sslServerSock = (SSLServerSocket) serverSock;
- SSLParameters sslParameters = sslServerSock.getSSLParameters();
- sslServerSock.setEnabledCipherSuites(getFilteredCipherSuites(sslParameters));
- sslServerSock.setEnabledProtocols(getFilteredProtocols(sslParameters));
- }
- return transport;
- } catch (Throwable throwable) {
- throw new FlumeException("Cannot start Thrift source.", throwable);
- }
- }
-
- private TServerTransport getTServerTransport() {
- try {
- return new TServerSocket(new InetSocketAddress(bindAddress, port));
- } catch (Throwable throwable) {
- throw new FlumeException("Cannot start Thrift source.", throwable);
- }
- }
-
- private TProtocolFactory getProtocolFactory() {
- if (protocol.equals(BINARY_PROTOCOL)) {
- logger.info("Using TBinaryProtocol");
- return new TBinaryProtocol.Factory();
- } else {
- logger.info("Using TCompactProtocol");
- return new TCompactProtocol.Factory();
- }
- }
-
- private TServer getTThreadedSelectorServer() {
- if (isSslEnabled() || enableKerberos) {
- return null;
- }
- Class<?> serverClass;
- Class<?> argsClass;
- TServer.AbstractServerArgs args;
- try {
- serverClass = Class.forName("org.apache.thrift" +
- ".server.TThreadedSelectorServer");
- argsClass = Class.forName("org.apache.thrift" +
- ".server.TThreadedSelectorServer$Args");
-
- TServerTransport serverTransport = new TNonblockingServerSocket(
- new InetSocketAddress(bindAddress, port));
-
- ExecutorService sourceService;
- ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat(
- "Flume Thrift IPC Thread %d").build();
- if (maxThreads == 0) {
- sourceService = Executors.newCachedThreadPool(threadFactory);
- } else {
- sourceService = Executors.newFixedThreadPool(maxThreads, threadFactory);
- }
- args = (TNonblockingServer.AbstractNonblockingServerArgs) argsClass
- .getConstructor(TNonblockingServerTransport.class)
- .newInstance(serverTransport);
- Method m = argsClass.getDeclaredMethod("executorService",
- ExecutorService.class);
- m.invoke(args, sourceService);
-
- populateServerParams(args);
-
- /*
- * Both THsHaServer and TThreadedSelectorServer allows us to pass in
- * the executor service to use - unfortunately the "executorService"
- * method does not exist in the parent abstract Args class,
- * so use reflection to pass the executor in.
- *
- */
- server = (TServer) serverClass.getConstructor(argsClass).newInstance(args);
- } catch (ClassNotFoundException e) {
- return null;
- } catch (Throwable ex) {
- throw new FlumeException("Cannot start Thrift Source.", ex);
- }
- return server;
- }
-
- private TServer getTThreadPoolServer() {
- TServerTransport serverTransport;
- if (isSslEnabled()) {
- serverTransport = getSSLServerTransport();
- } else {
- serverTransport = getTServerTransport();
- }
- TThreadPoolServer.Args serverArgs = new TThreadPoolServer.Args(serverTransport);
- serverArgs.maxWorkerThreads(maxThreads);
- populateServerParams(serverArgs);
- return new TThreadPoolServer(serverArgs);
- }
-
- private void populateServerParams(TServer.AbstractServerArgs args) {
- //populate the ProtocolFactory
- args.protocolFactory(getProtocolFactory());
-
- //populate the transportFactory
- if (enableKerberos) {
- args.transportFactory(getSASLTransportFactory());
- } else {
- args.transportFactory(new TFastFramedTransport.Factory());
- }
-
- // populate the Processor
- args.processor(new ThriftSourceProtocol
- .Processor<ThriftSourceHandler>(new ThriftSourceHandler()));
- }
-
- private TTransportFactory getSASLTransportFactory() {
- String[] names;
- try {
- names = FlumeAuthenticationUtil.splitKerberosName(principal);
- } catch (IOException e) {
- throw new FlumeException(
- "Error while trying to resolve Principal name - " + principal, e);
- }
- Map<String, String> saslProperties = new HashMap<String, String>();
- saslProperties.put(Sasl.QOP, "auth");
- TSaslServerTransport.Factory saslTransportFactory =
- new TSaslServerTransport.Factory();
- saslTransportFactory.addServerDefinition(
- "GSSAPI", names[0], names[1], saslProperties,
- FlumeAuthenticationUtil.getSaslGssCallbackHandler());
- return saslTransportFactory;
- }
-
- @Override
- public void stop() {
- if (server != null && server.isServing()) {
- server.stop();
- }
- if (servingExecutor != null) {
- servingExecutor.shutdown();
- try {
- if (!servingExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
- servingExecutor.shutdownNow();
- }
- } catch (InterruptedException e) {
- throw new FlumeException("Interrupted while waiting for server to be " +
- "shutdown.");
- }
- }
- sourceCounter.stop();
- super.stop();
- }
-
- private class ThriftSourceHandler implements ThriftSourceProtocol.Iface {
-
- @Override
- public Status append(ThriftFlumeEvent event) throws TException {
- Event flumeEvent = EventBuilder.withBody(event.getBody(), event.getHeaders());
-
- sourceCounter.incrementAppendReceivedCount();
- sourceCounter.incrementEventReceivedCount();
-
- try {
- getChannelProcessor().processEvent(flumeEvent);
- } catch (ChannelException ex) {
- logger.warn("Thrift source " + getName() + " could not append events " +
- "to the channel.", ex);
- sourceCounter.incrementChannelWriteFail();
- return Status.FAILED;
- }
- sourceCounter.incrementAppendAcceptedCount();
- sourceCounter.incrementEventAcceptedCount();
- return Status.OK;
- }
-
- @Override
- public Status appendBatch(List<ThriftFlumeEvent> events) throws TException {
- sourceCounter.incrementAppendBatchReceivedCount();
- sourceCounter.addToEventReceivedCount(events.size());
-
- List<Event> flumeEvents = Lists.newArrayList();
- for (ThriftFlumeEvent event : events) {
- flumeEvents.add(EventBuilder.withBody(event.getBody(), event.getHeaders()));
- }
-
- try {
- getChannelProcessor().processEventBatch(flumeEvents);
- } catch (ChannelException ex) {
- logger.warn("Thrift source %s could not append events to the channel.", getName());
- sourceCounter.incrementChannelWriteFail();
- return Status.FAILED;
- }
-
- sourceCounter.incrementAppendBatchAcceptedCount();
- sourceCounter.addToEventAcceptedCount(events.size());
- return Status.OK;
- }
- }
-}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/sink/TestAvroSink.java b/flume-ng-core/src/test/java/org/apache/flume/sink/TestAvroSink.java
deleted file mode 100644
index 30bc5f6b5..000000000
--- a/flume-ng-core/src/test/java/org/apache/flume/sink/TestAvroSink.java
+++ /dev/null
@@ -1,913 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.sink;
-
-import com.google.common.base.Charsets;
-import org.apache.avro.ipc.netty.NettyServer;
-import org.apache.avro.ipc.Server;
-import org.apache.avro.ipc.specific.SpecificResponder;
-import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
-import org.apache.flume.ChannelSelector;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.Sink;
-import org.apache.flume.Transaction;
-import org.apache.flume.api.RpcClient;
-import org.apache.flume.channel.BasicTransactionSemantics;
-import org.apache.flume.channel.ChannelProcessor;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.channel.ReplicatingChannelSelector;
-import org.apache.flume.conf.Configurables;
-import org.apache.flume.event.EventBuilder;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.flume.lifecycle.LifecycleController;
-import org.apache.flume.lifecycle.LifecycleState;
-import org.apache.flume.source.AvroSource;
-import org.apache.flume.source.avro.AvroFlumeEvent;
-import org.apache.flume.source.avro.AvroSourceProtocol;
-import org.apache.flume.source.avro.Status;
-import org.junit.Assert;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.mockito.internal.util.reflection.Whitebox;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLEngine;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.nio.charset.Charset;
-import java.security.KeyStore;
-import java.security.Security;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-
-import io.netty.channel.ChannelPipeline;
-import io.netty.handler.ssl.SslHandler;
-
-public class TestAvroSink {
-
- private static final Logger logger = LoggerFactory
- .getLogger(TestAvroSink.class);
- private static final String hostname = "127.0.0.1";
- private static final Integer port;
-
- static {
- try (ServerSocket socket = new ServerSocket(0)) {
- port = socket.getLocalPort();
- } catch (IOException e) {
- throw new AssertionError("Cannot find free port", e);
- }
- }
-
- private AvroSink sink;
- private Channel channel;
-
-
- public void setUp() {
- setUp("none", 0);
- }
-
- public void setUp(String compressionType, int compressionLevel) {
- if (sink != null) {
- throw new RuntimeException("double setup");
- }
- sink = new AvroSink();
- channel = new MemoryChannel();
-
- Context context = createBaseContext();
-
- if (compressionType.equals("deflate")) {
- context.put("compression-type", compressionType);
- context.put("compression-level", Integer.toString(compressionLevel));
- }
-
- sink.setChannel(channel);
-
- Configurables.configure(sink, context);
- Configurables.configure(channel, context);
- }
-
- private Context createBaseContext() {
- Context context = new Context();
-
- context.put("hostname", hostname);
- context.put("port", String.valueOf(port));
- context.put("batch-size", String.valueOf(2));
- context.put("connect-timeout", String.valueOf(2000L));
- context.put("request-timeout", String.valueOf(3000L));
-
- return context;
- }
-
- private Server createServer(AvroSourceProtocol protocol) throws InterruptedException {
-
- Server server = new NettyServer(new SpecificResponder(AvroSourceProtocol.class, protocol),
- new InetSocketAddress(hostname, port));
-
- return server;
- }
-
- @Test
- public void testLifecycle() throws InterruptedException,
- InstantiationException, IllegalAccessException {
- setUp();
- Server server = createServer(new MockAvroServer());
-
- server.start();
-
- sink.start();
- Assert.assertTrue(LifecycleController.waitForOneOf(sink,
- LifecycleState.START_OR_ERROR, 5000));
-
- sink.stop();
- Assert.assertTrue(LifecycleController.waitForOneOf(sink,
- LifecycleState.STOP_OR_ERROR, 5000));
-
- server.close();
- }
-
- @Test
- public void testProcess() throws InterruptedException,
- EventDeliveryException, InstantiationException, IllegalAccessException {
- setUp();
-
- Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
- Server server = createServer(new MockAvroServer());
-
- server.start();
-
- sink.start();
- Assert.assertTrue(LifecycleController.waitForOneOf(sink,
- LifecycleState.START_OR_ERROR, 5000));
-
- Transaction transaction = channel.getTransaction();
-
- transaction.begin();
- for (int i = 0; i < 10; i++) {
- channel.put(event);
- }
- transaction.commit();
- transaction.close();
-
- for (int i = 0; i < 5; i++) {
- Sink.Status status = sink.process();
- Assert.assertEquals(Sink.Status.READY, status);
- }
-
- Assert.assertEquals(Sink.Status.BACKOFF, sink.process());
-
- sink.stop();
- Assert.assertTrue(LifecycleController.waitForOneOf(sink,
- LifecycleState.STOP_OR_ERROR, 5000));
-
- server.close();
- }
-
- @Test
- public void testChannelException() throws InterruptedException,
- EventDeliveryException, InstantiationException, IllegalAccessException {
- setUp();
-
- Server server = createServer(new MockAvroServer());
- server.start();
- sink.start();
- Channel channel = Mockito.mock(Channel.class);
- Mockito.when(channel.take()).thenThrow(new ChannelException("dummy"));
- Transaction transaction = Mockito.mock(BasicTransactionSemantics.class);
- Mockito.when(channel.getTransaction()).thenReturn(transaction);
- sink.setChannel(channel);
-
- Sink.Status status = sink.process();
-
- sink.stop();
- server.close();
-
- SinkCounter sinkCounter = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
- Assert.assertEquals(1, sinkCounter.getChannelReadFail());
- }
-
- @Test
- public void testTimeout() throws InterruptedException,
- EventDeliveryException, InstantiationException, IllegalAccessException {
- setUp();
- Event event = EventBuilder.withBody("foo", Charsets.UTF_8);
- AtomicLong delay = new AtomicLong();
- Server server = createServer(new DelayMockAvroServer(delay));
- server.start();
- sink.start();
- Assert.assertTrue(LifecycleController.waitForOneOf(sink,
- LifecycleState.START_OR_ERROR, 5000));
-
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (int i = 0; i < 4; i++) {
- channel.put(event);
- }
- txn.commit();
- txn.close();
-
- // should throw EventDeliveryException due to connect timeout
- delay.set(3000L); // because connect-timeout = 2000
- boolean threw = false;
- try {
- sink.process();
- } catch (EventDeliveryException ex) {
- logger.info("Correctly threw due to connect timeout. Exception follows.",
- ex);
- threw = true;
- }
-
- Assert.assertTrue("Must throw due to connect timeout", threw);
-
- // now, allow the connect handshake to occur
- delay.set(0);
- sink.process();
-
- // should throw another EventDeliveryException due to request timeout
- delay.set(4000L); // because request-timeout = 3000
- threw = false;
- try {
- sink.process();
- } catch (EventDeliveryException ex) {
- logger.info("Correctly threw due to request timeout. Exception follows.",
- ex);
- threw = true;
- }
-
- Assert.assertTrue("Must throw due to request timeout", threw);
-
- sink.stop();
- Assert.assertTrue(LifecycleController.waitForOneOf(sink,
- LifecycleState.STOP_OR_ERROR, 5000));
- server.close();
-
- SinkCounter sinkCounter = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
- Assert.assertEquals(2, sinkCounter.getEventWriteFail());
- }
-
- @Test
- public void testFailedConnect() throws InterruptedException,
- EventDeliveryException, InstantiationException, IllegalAccessException {
-
- setUp();
- Event event = EventBuilder.withBody("test event 1",
- Charset.forName("UTF8"));
- Server server = createServer(new MockAvroServer());
-
- server.start();
- sink.start();
- Assert.assertTrue(LifecycleController.waitForOneOf(sink,
- LifecycleState.START_OR_ERROR, 5000));
-
- Thread.sleep(500L); // let socket startup
- server.close();
- Thread.sleep(500L); // sleep a little to allow close occur
-
- Transaction transaction = channel.getTransaction();
-
- transaction.begin();
- for (int i = 0; i < 10; i++) {
- channel.put(event);
- }
- transaction.commit();
- transaction.close();
-
- for (int i = 0; i < 5; i++) {
- boolean threwException = false;
- try {
- sink.process();
- } catch (EventDeliveryException e) {
- threwException = true;
- }
- Assert.assertTrue("Must throw EventDeliveryException if disconnected",
- threwException);
- }
-
- server = createServer(new MockAvroServer());
- server.start();
-
- for (int i = 0; i < 5; i++) {
- Sink.Status status = sink.process();
- Assert.assertEquals(Sink.Status.READY, status);
- }
-
- Assert.assertEquals(Sink.Status.BACKOFF, sink.process());
-
- sink.stop();
- Assert.assertTrue(LifecycleController.waitForOneOf(sink,
- LifecycleState.STOP_OR_ERROR, 5000));
- server.close();
-
- SinkCounter sinkCounter = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
- Assert.assertEquals(5, sinkCounter.getEventWriteFail());
- Assert.assertEquals(4, sinkCounter.getConnectionFailedCount());
- }
-
- @Test
- public void testReset() throws Exception {
-
- setUp();
- Server server = createServer(new MockAvroServer());
-
- server.start();
-
- Context context = new Context();
-
- context.put("hostname", hostname);
- context.put("port", String.valueOf(port));
- context.put("batch-size", String.valueOf(2));
- context.put("connect-timeout", String.valueOf(2000L));
- context.put("request-timeout", String.valueOf(3000L));
- context.put("reset-connection-interval", String.valueOf("5"));
-
- sink.setChannel(channel);
- Configurables.configure(sink, context);
- sink.start();
- RpcClient firstClient = sink.getUnderlyingClient();
- Thread.sleep(6000);
- Transaction t = channel.getTransaction();
- t.begin();
- channel.put(EventBuilder.withBody("This is a test", Charset.defaultCharset()));
- t.commit();
- t.close();
- sink.process();
- // Make sure they are not the same object, connection should be reset
- Assert.assertFalse(firstClient == sink.getUnderlyingClient());
- sink.stop();
-
- context.put("hostname", hostname);
- context.put("port", String.valueOf(port));
- context.put("batch-size", String.valueOf(2));
- context.put("connect-timeout", String.valueOf(2000L));
- context.put("request-timeout", String.valueOf(3000L));
- context.put("reset-connection-interval", String.valueOf("0"));
-
- sink.setChannel(channel);
- Configurables.configure(sink, context);
- sink.start();
- firstClient = sink.getUnderlyingClient();
- Thread.sleep(6000);
- // Make sure they are the same object, since connection should not be reset
- Assert.assertTrue(firstClient == sink.getUnderlyingClient());
- sink.stop();
-
- context.clear();
- context.put("hostname", hostname);
- context.put("port", String.valueOf(port));
- context.put("batch-size", String.valueOf(2));
- context.put("connect-timeout", String.valueOf(2000L));
- context.put("request-timeout", String.valueOf(3000L));
-
- sink.setChannel(channel);
- Configurables.configure(sink, context);
- sink.start();
- firstClient = sink.getUnderlyingClient();
- Thread.sleep(6000);
- // Make sure they are the same object, since connection should not be reset
- Assert.assertTrue(firstClient == sink.getUnderlyingClient());
- sink.stop();
- server.close();
- }
-
- @Test
- public void testSslProcessTrustAllCerts() throws InterruptedException,
- EventDeliveryException, InstantiationException, IllegalAccessException {
- setUp();
-
- Context context = createBaseContext();
- context.put("ssl", String.valueOf(true));
- context.put("trust-all-certs", String.valueOf(true));
-
- Configurables.configure(sink, context);
-
- doTestSslProcess();
- }
-
- @Test
- public void testSslProcessWithComponentTruststore() throws InterruptedException,
- EventDeliveryException, InstantiationException, IllegalAccessException {
- setUp();
-
- Context context = createBaseContext();
- context.put("ssl", String.valueOf(true));
- context.put("truststore", "src/test/resources/truststore.jks");
- context.put("truststore-password", "password");
-
- Configurables.configure(sink, context);
-
- doTestSslProcess();
- }
-
- @Test
- public void testSslProcessWithComponentTruststoreNoPassword() throws InterruptedException,
- EventDeliveryException, InstantiationException, IllegalAccessException {
- setUp();
-
- Context context = createBaseContext();
- context.put("ssl", String.valueOf(true));
- context.put("truststore", "src/test/resources/truststore.jks");
-
- Configurables.configure(sink, context);
-
- doTestSslProcess();
- }
-
- @Test
- public void testSslProcessWithGlobalTruststore() throws InterruptedException,
- EventDeliveryException, InstantiationException, IllegalAccessException {
- setUp();
-
- System.setProperty("javax.net.ssl.trustStore", "src/test/resources/truststore.jks");
- System.setProperty("javax.net.ssl.trustStorePassword", "password");
-
- Context context = createBaseContext();
- context.put("ssl", String.valueOf(true));
-
- Configurables.configure(sink, context);
-
- doTestSslProcess();
-
- System.clearProperty("javax.net.ssl.trustStore");
- System.clearProperty("javax.net.ssl.trustStorePassword");
- }
-
- @Test
- public void testSslProcessWithGlobalTruststoreNoPassword() throws InterruptedException,
- EventDeliveryException, InstantiationException, IllegalAccessException {
- setUp();
-
- System.setProperty("javax.net.ssl.trustStore", "src/test/resources/truststore.jks");
-
- Context context = createBaseContext();
- context.put("ssl", String.valueOf(true));
-
- Configurables.configure(sink, context);
-
- doTestSslProcess();
-
- System.clearProperty("javax.net.ssl.trustStore");
- }
-
- private void doTestSslProcess() throws InterruptedException,
- EventDeliveryException, InstantiationException, IllegalAccessException {
- Server server = createSslServer(new MockAvroServer());
- server.start();
-
- sink.start();
- Assert.assertTrue(LifecycleController.waitForOneOf(sink,
- LifecycleState.START_OR_ERROR, 5000));
-
- Transaction transaction = channel.getTransaction();
- transaction.begin();
-
- Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
- for (int i = 0; i < 10; i++) {
- channel.put(event);
- }
-
- transaction.commit();
- transaction.close();
-
- for (int i = 0; i < 5; i++) {
- Sink.Status status = sink.process();
- Assert.assertEquals(Sink.Status.READY, status);
- }
-
- Assert.assertEquals(Sink.Status.BACKOFF, sink.process());
-
- sink.stop();
- Assert.assertTrue(LifecycleController.waitForOneOf(sink,
- LifecycleState.STOP_OR_ERROR, 5000));
-
- server.close();
- }
-
- @Test
- public void testSslWithCompression() throws InterruptedException,
- EventDeliveryException, InstantiationException, IllegalAccessException {
- setUp("deflate", 6);
-
- boolean bound = false;
-
- AvroSource source;
- Channel sourceChannel;
- int selectedPort;
-
- source = new AvroSource();
- sourceChannel = new MemoryChannel();
-
- Configurables.configure(sourceChannel, new Context());
-
- List<Channel> channels = new ArrayList<Channel>();
- channels.add(sourceChannel);
-
- ChannelSelector rcs = new ReplicatingChannelSelector();
- rcs.setChannels(channels);
-
- source.setChannelProcessor(new ChannelProcessor(rcs));
-
- Context context = new Context();
- context.put("port", port.toString());
- context.put("bind", hostname);
- context.put("threads", "50");
- context.put("compression-type", "deflate");
- context.put("ssl", String.valueOf(true));
- context.put("keystore", "src/test/resources/server.p12");
- context.put("keystore-password", "password");
- context.put("keystore-type", "PKCS12");
-
- Configurables.configure(source, context);
-
- source.start();
-
- Assert
- .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
- source, LifecycleState.START_OR_ERROR));
- Assert.assertEquals("Server is started", LifecycleState.START,
- source.getLifecycleState());
-
-
- Event event = EventBuilder.withBody("Hello avro",
- Charset.forName("UTF8"));
-
- context = createBaseContext();
- context.put("ssl", String.valueOf(true));
- context.put("trust-all-certs", String.valueOf(true));
- context.put("compression-type", "deflate");
- context.put("compression-level", Integer.toString(6));
-
- Configurables.configure(sink, context);
- sink.start();
-
- Transaction sickTransaction = channel.getTransaction();
-
- sickTransaction.begin();
- for (int i = 0; i < 10; i++) {
- channel.put(event);
- }
- sickTransaction.commit();
- sickTransaction.close();
-
- for (int i = 0; i < 5; i++) {
- Sink.Status status = sink.process();
- logger.debug("Calling Process " + i + " times:" + status);
- Assert.assertEquals(Sink.Status.READY, status);
- }
-
- sink.stop();
-
- Transaction sourceTransaction = sourceChannel.getTransaction();
- sourceTransaction.begin();
-
- Event sourceEvent = sourceChannel.take();
- Assert.assertNotNull(sourceEvent);
- Assert.assertEquals("Channel contained our event", "Hello avro",
- new String(sourceEvent.getBody()));
- sourceTransaction.commit();
- sourceTransaction.close();
-
- logger.debug("Round trip event:{}", sourceEvent);
-
- source.stop();
- Assert.assertTrue("Reached stop or error",
- LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
- Assert.assertEquals("Server is stopped", LifecycleState.STOP,
- source.getLifecycleState());
- }
-
- @Test
- public void testSslSinkWithNonSslServer() throws InterruptedException,
- InstantiationException, IllegalAccessException {
- setUp();
-
- Server server = createServer(new MockAvroServer());
- server.start();
-
- Context context = createBaseContext();
- context.put("ssl", String.valueOf(true));
- context.put("trust-all-certs", String.valueOf(true));
-
- Configurables.configure(sink, context);
-
- boolean failed = doRequestWhenFailureExpected();
-
- server.close();
-
- if (!failed) {
- Assert.fail("SSL-enabled sink successfully connected to a non-SSL-enabled server, " +
- "that's wrong.");
- }
-
- SinkCounter sinkCounter = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
- Assert.assertEquals(1, sinkCounter.getEventWriteFail());
- }
-
- @Test
- public void testSslSinkWithNonTrustedCert() throws InterruptedException,
- InstantiationException, IllegalAccessException {
- setUp();
-
- Server server = createSslServer(new MockAvroServer());
- server.start();
-
- Context context = createBaseContext();
- context.put("ssl", String.valueOf(true));
-
- Configurables.configure(sink, context);
-
- boolean failed = doRequestWhenFailureExpected();
-
- server.close();
-
- if (!failed) {
- Assert.fail("SSL-enabled sink successfully connected to a server with an " +
- "untrusted certificate when it should have failed");
- }
- SinkCounter sinkCounter = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
- Assert.assertEquals(1, sinkCounter.getEventWriteFail());
- }
-
- private boolean doRequestWhenFailureExpected()
- throws InterruptedException {
- sink.start();
- Assert.assertTrue(LifecycleController.waitForOneOf(sink,
- LifecycleState.START_OR_ERROR, 5000));
-
- Transaction transaction = channel.getTransaction();
- transaction.begin();
-
- Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
- channel.put(event);
-
- transaction.commit();
- transaction.close();
-
- boolean failed;
- try {
- sink.process();
- failed = false;
- } catch (EventDeliveryException ex) {
- logger.info("Correctly failed to send event", ex);
- failed = true;
- }
-
- sink.stop();
- Assert.assertTrue(LifecycleController.waitForOneOf(sink,
- LifecycleState.STOP_OR_ERROR, 5000));
-
- return failed;
- }
-
- @Test
- public void testRequestWithNoCompression()
- throws InterruptedException, IOException, EventDeliveryException {
- doRequest(false, false, 6);
- }
-
- @Test
- public void testRequestWithCompressionOnClientAndServerOnLevel0()
- throws InterruptedException, IOException, EventDeliveryException {
- doRequest(true, true, 0);
- }
-
- @Test
- public void testRequestWithCompressionOnClientAndServerOnLevel1()
- throws InterruptedException, IOException, EventDeliveryException {
- doRequest(true, true, 1);
- }
-
- @Test
- public void testRequestWithCompressionOnClientAndServerOnLevel6()
- throws InterruptedException, IOException, EventDeliveryException {
- doRequest(true, true, 6);
- }
-
- @Test
- public void testRequestWithCompressionOnClientAndServerOnLevel9()
- throws InterruptedException, IOException, EventDeliveryException {
- doRequest(true, true, 9);
- }
-
- private void doRequest(boolean serverEnableCompression, boolean clientEnableCompression,
- int compressionLevel)
- throws InterruptedException, IOException, EventDeliveryException {
- if (clientEnableCompression) {
- setUp("deflate", compressionLevel);
- } else {
- setUp("none", compressionLevel);
- }
-
- boolean bound = false;
-
- AvroSource source;
- Channel sourceChannel;
- int selectedPort;
-
- source = new AvroSource();
- sourceChannel = new MemoryChannel();
-
- Configurables.configure(sourceChannel, new Context());
-
- List<Channel> channels = new ArrayList<Channel>();
- channels.add(sourceChannel);
-
- ChannelSelector rcs = new ReplicatingChannelSelector();
- rcs.setChannels(channels);
-
- source.setChannelProcessor(new ChannelProcessor(rcs));
-
- Context context = new Context();
- context.put("port", port.toString());
- context.put("bind", hostname);
- context.put("threads", "50");
- if (serverEnableCompression) {
- context.put("compression-type", "deflate");
- } else {
- context.put("compression-type", "none");
- }
-
- Configurables.configure(source, context);
-
- source.start();
-
- Assert.assertTrue("Reached start or error",
- LifecycleController.waitForOneOf(source, LifecycleState.START_OR_ERROR));
- Assert.assertEquals("Server is started",
- LifecycleState.START, source.getLifecycleState());
-
- Event event = EventBuilder.withBody("Hello avro", Charset.forName("UTF8"));
-
- sink.start();
-
- Transaction sickTransaction = channel.getTransaction();
-
- sickTransaction.begin();
- for (int i = 0; i < 10; i++) {
- channel.put(event);
- }
- sickTransaction.commit();
- sickTransaction.close();
-
- for (int i = 0; i < 5; i++) {
- Sink.Status status = sink.process();
- logger.debug("Calling Process " + i + " times:" + status);
- Assert.assertEquals(Sink.Status.READY, status);
- }
-
- sink.stop();
-
-
- Transaction sourceTransaction = sourceChannel.getTransaction();
- sourceTransaction.begin();
-
- Event sourceEvent = sourceChannel.take();
- Assert.assertNotNull(sourceEvent);
- Assert.assertEquals("Channel contained our event", "Hello avro",
- new String(sourceEvent.getBody()));
- sourceTransaction.commit();
- sourceTransaction.close();
-
- logger.debug("Round trip event:{}", sourceEvent);
-
- source.stop();
- Assert.assertTrue("Reached stop or error",
- LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
- Assert.assertEquals("Server is stopped", LifecycleState.STOP,
- source.getLifecycleState());
- }
-
- private static class MockAvroServer implements AvroSourceProtocol {
-
- @Override
- public Status append(AvroFlumeEvent event) {
- logger.debug("Received event:{}", event);
- return Status.OK;
- }
-
- @Override
- public Status appendBatch(List<AvroFlumeEvent> events) {
- logger.debug("Received event batch:{}", events);
- return Status.OK;
- }
-
- }
-
- private static class DelayMockAvroServer implements AvroSourceProtocol.Callback {
-
- private final AtomicLong delay;
-
- public DelayMockAvroServer(AtomicLong delay) {
- this.delay = delay;
- }
-
- private void sleep() throws IOException {
- try {
- Thread.sleep(delay.get());
- } catch (InterruptedException e) {
- throw new IOException("Interrupted while sleeping", e);
- }
- }
-
- @Override
- public Status append(AvroFlumeEvent event) {
- logger.debug("Received event:{}; delaying for {}ms", event, delay);
- try {
- sleep();
- } catch (IOException e) {
- logger.debug("IOException detected");
- }
- return Status.OK;
- }
-
- @Override
- public void append(AvroFlumeEvent event,
- org.apache.avro.ipc.Callback<Status> status)
- throws IOException {
- logger.debug("Received event:{}; delaying for {}ms", event, delay);
- sleep();
- }
-
- @Override
- public Status appendBatch(List<AvroFlumeEvent> events) {
- logger.debug("Received event batch:{}; delaying for {}ms", events, delay);
- try {
- sleep();
- } catch (IOException e) {
- logger.debug("IOException detected");
- }
- return Status.OK;
- }
-
- @Override
- public void appendBatch(List<AvroFlumeEvent> events,
- org.apache.avro.ipc.Callback<Status> status)
- throws IOException {
- logger.debug("Received event batch:{}; delaying for {}ms", events, delay);
- sleep();
- }
- }
-
- private Server createSslServer(AvroSourceProtocol protocol) throws InterruptedException {
- return new NettyServer(new SpecificResponder(AvroSourceProtocol.class, protocol),
- new InetSocketAddress(hostname, port),
- (ch) -> {
- ChannelPipeline pipeline = ch.pipeline();
- SSLEngine engine = createSSLEngine();
- engine.setUseClientMode(false);
- pipeline.addLast("ssl", new SslHandler(engine));
- });
- }
-
- private SSLEngine createSSLEngine() {
- String keystore = "src/test/resources/server.p12";
- String keystorePassword = "password";
- String keystoreType = "PKCS12";
- try {
- KeyStore ks = KeyStore.getInstance(keystoreType);
- ks.load(new FileInputStream(keystore), keystorePassword.toCharArray());
-
- // Set up key manager factory to use our key store
- KeyManagerFactory kmf = KeyManagerFactory.getInstance(getAlgorithm());
- kmf.init(ks, keystorePassword.toCharArray());
-
- SSLContext sslContext = SSLContext.getInstance("TLS");
- sslContext.init(kmf.getKeyManagers(), null, null);
- return sslContext.createSSLEngine();
- } catch (Exception ex) {
- throw new RuntimeException("Cannot create SSL Engine", ex);
- }
- }
-
- private String getAlgorithm() {
- String algorithm = Security.getProperty("ssl.KeyManagerFactory.algorithm");
- if (algorithm == null) {
- algorithm = "SunX509";
- }
- return algorithm;
- }
-}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/sink/TestThriftSink.java b/flume-ng-core/src/test/java/org/apache/flume/sink/TestThriftSink.java
deleted file mode 100644
index a39bbc822..000000000
--- a/flume-ng-core/src/test/java/org/apache/flume/sink/TestThriftSink.java
+++ /dev/null
@@ -1,366 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink;
-
-import com.google.common.base.Charsets;
-
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.Sink;
-import org.apache.flume.Transaction;
-import org.apache.flume.api.ThriftRpcClient;
-import org.apache.flume.api.ThriftTestingSource;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.conf.Configurables;
-import org.apache.flume.event.EventBuilder;
-import org.apache.flume.lifecycle.LifecycleController;
-import org.apache.flume.lifecycle.LifecycleState;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.Ignore;
-
-import javax.net.ssl.KeyManagerFactory;
-
-import java.net.ServerSocket;
-import java.nio.charset.Charset;
-import java.util.concurrent.atomic.AtomicLong;
-
-public class TestThriftSink {
- private ThriftTestingSource src;
- private ThriftSink sink;
- private MemoryChannel channel;
- private String hostname;
- private int port;
-
- @Before
- public void setUp() throws Exception {
- sink = new ThriftSink();
- channel = new MemoryChannel();
- hostname = "0.0.0.0";
- try (ServerSocket socket = new ServerSocket(0)) {
- port = socket.getLocalPort();
- }
- Context context = createBaseContext();
- context.put(ThriftRpcClient.CONFIG_PROTOCOL, ThriftRpcClient.COMPACT_PROTOCOL);
- sink.setChannel(channel);
-
- Configurables.configure(sink, context);
- Configurables.configure(channel, context);
- }
-
- private Context createBaseContext() {
- Context context = new Context();
-
- context.put("hostname", hostname);
- context.put("port", String.valueOf(port));
- context.put("batch-size", String.valueOf(2));
- context.put("connect-timeout", String.valueOf(2000L));
- context.put("request-timeout", String.valueOf(2000L));
-
- return context;
- }
-
- @After
- public void tearDown() throws Exception {
- channel.stop();
- sink.stop();
- src.stop();
- }
-
- @Test
- public void testProcess() throws Exception {
-
- Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
- src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(),
- port, ThriftRpcClient.COMPACT_PROTOCOL);
-
- channel.start();
- sink.start();
-
- Transaction transaction = channel.getTransaction();
-
- transaction.begin();
- for (int i = 0; i < 11; i++) {
- channel.put(event);
- }
- transaction.commit();
- transaction.close();
- for (int i = 0; i < 6; i++) {
- Sink.Status status = sink.process();
- Assert.assertEquals(Sink.Status.READY, status);
- }
-
- Assert.assertEquals(Sink.Status.BACKOFF, sink.process());
-
- sink.stop();
- Assert.assertEquals(11, src.flumeEvents.size());
- Assert.assertEquals(6, src.batchCount);
- Assert.assertEquals(0, src.individualCount);
-
- }
-
- @Test
- public void testTimeout() throws Exception {
- AtomicLong delay = new AtomicLong();
- src = new ThriftTestingSource(ThriftTestingSource.HandlerType.ALTERNATE
- .name(), port, ThriftRpcClient.COMPACT_PROTOCOL);
- src.setDelay(delay);
- delay.set(2500);
-
- Event event = EventBuilder.withBody("foo", Charsets.UTF_8);
- sink.start();
-
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (int i = 0; i < 4; i++) {
- channel.put(event);
- }
- txn.commit();
- txn.close();
-
- // should throw EventDeliveryException due to connect timeout
- boolean threw = false;
- try {
- sink.process();
- } catch (EventDeliveryException ex) {
- threw = true;
- }
-
- Assert.assertTrue("Must throw due to connect timeout", threw);
-
- // now, allow the connect handshake to occur
- delay.set(0);
- sink.process();
-
- // should throw another EventDeliveryException due to request timeout
- delay.set(2500L); // because request-timeout = 2000
- threw = false;
- try {
- sink.process();
- } catch (EventDeliveryException ex) {
- threw = true;
- }
-
- Assert.assertTrue("Must throw due to request timeout", threw);
-
- sink.stop();
- }
-
- @Test
- public void testFailedConnect() throws Exception {
-
- Event event = EventBuilder.withBody("test event 1", Charset.forName("UTF8"));
-
- sink.start();
-
- Thread.sleep(500L); // let socket startup
- Thread.sleep(500L); // sleep a little to allow close occur
-
- Transaction transaction = channel.getTransaction();
- transaction.begin();
- for (int i = 0; i < 10; i++) {
- channel.put(event);
- }
- transaction.commit();
- transaction.close();
-
- for (int i = 0; i < 5; i++) {
- boolean threwException = false;
- try {
- sink.process();
- } catch (EventDeliveryException e) {
- threwException = true;
- }
- Assert.assertTrue("Must throw EventDeliveryException if disconnected",
- threwException);
- }
-
- src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(),
- port, ThriftRpcClient.COMPACT_PROTOCOL);
-
- for (int i = 0; i < 5; i++) {
- Sink.Status status = sink.process();
- Assert.assertEquals(Sink.Status.READY, status);
- }
-
- Assert.assertEquals(Sink.Status.BACKOFF, sink.process());
- sink.stop();
- }
-
- @Ignore("This test is flakey and causes tests to fail pretty often.")
- @Test
- public void testSslProcessWithComponentTruststore() throws Exception {
- Context context = createBaseContext();
- context.put("ssl", String.valueOf(true));
- context.put("truststore", "src/test/resources/truststorefile.jks");
- context.put("truststore-password", "password");
-
- Configurables.configure(sink, context);
-
- doTestSslProcess();
- }
-
- @Ignore("This test is flakey and causes tests to fail pretty often.")
- @Test
- public void testSslProcessWithComponentTruststoreNoPassword() throws Exception {
- Context context = createBaseContext();
- context.put("ssl", String.valueOf(true));
- context.put("truststore", "src/test/resources/truststorefile.jks");
-
- Configurables.configure(sink, context);
-
- doTestSslProcess();
- }
-
- @Ignore("This test is flakey and causes tests to fail pretty often.")
- @Test
- public void testSslProcessWithGlobalTruststore() throws Exception {
- System.setProperty("javax.net.ssl.trustStore", "src/test/resources/truststorefile.jks");
- System.setProperty("javax.net.ssl.trustStorePassword", "password");
-
- Context context = createBaseContext();
- context.put("ssl", String.valueOf(true));
-
- Configurables.configure(sink, context);
-
- doTestSslProcess();
-
- System.clearProperty("javax.net.ssl.trustStore");
- System.clearProperty("javax.net.ssl.trustStorePassword");
- }
-
- @Ignore("This test is flakey and causes tests to fail pretty often.")
- @Test
- public void testSslProcessWithGlobalTruststoreNoPassword() throws Exception {
- System.setProperty("javax.net.ssl.trustStore", "src/test/resources/truststorefile.jks");
-
- Context context = createBaseContext();
- context.put("ssl", String.valueOf(true));
-
- Configurables.configure(sink, context);
-
- doTestSslProcess();
-
- System.clearProperty("javax.net.ssl.trustStore");
- }
-
- private void doTestSslProcess() throws Exception {
- src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(), port,
- ThriftRpcClient.COMPACT_PROTOCOL, "src/test/resources/keystorefile.jks",
- "password", KeyManagerFactory.getDefaultAlgorithm(), "JKS");
-
- channel.start();
- sink.start();
-
- Transaction transaction = channel.getTransaction();
- transaction.begin();
-
- Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
- for (int i = 0; i < 11; i++) {
- channel.put(event);
- }
-
- transaction.commit();
- transaction.close();
-
- for (int i = 0; i < 6; i++) {
- Sink.Status status = sink.process();
- Assert.assertEquals(Sink.Status.READY, status);
- }
- Assert.assertEquals(Sink.Status.BACKOFF, sink.process());
-
- sink.stop();
- Assert.assertEquals(11, src.flumeEvents.size());
- Assert.assertEquals(6, src.batchCount);
- Assert.assertEquals(0, src.individualCount);
- }
-
- @Test
- public void testSslSinkWithNonSslServer() throws Exception {
- src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(),
- port, ThriftRpcClient.COMPACT_PROTOCOL);
-
- Context context = createBaseContext();
- context.put("ssl", String.valueOf(true));
- context.put("truststore", "src/test/resources/truststorefile.jks");
- context.put("truststore-password", "password");
-
- Configurables.configure(sink, context);
-
- boolean failed = doRequestWhenFailureExpected();
- if (!failed) {
- Assert.fail("SSL-enabled sink successfully connected to a non-SSL-enabled server, " +
- "that's wrong.");
- }
- }
-
- @Test
- public void testSslSinkWithNonTrustedCert() throws Exception {
- src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(), port,
- ThriftRpcClient.COMPACT_PROTOCOL, "src/test/resources/keystorefile.jks",
- "password", KeyManagerFactory.getDefaultAlgorithm(), "JKS");
-
- Context context = createBaseContext();
- context.put("ssl", String.valueOf(true));
-
- Configurables.configure(sink, context);
-
- boolean failed = doRequestWhenFailureExpected();
- if (!failed) {
- Assert.fail("SSL-enabled sink successfully connected to a server with an " +
- "untrusted certificate when it should have failed");
- }
- }
-
- private boolean doRequestWhenFailureExpected() throws Exception {
- channel.start();
- sink.start();
- Assert.assertTrue(LifecycleController.waitForOneOf(sink,
- LifecycleState.START_OR_ERROR, 5000));
-
- Transaction transaction = channel.getTransaction();
- transaction.begin();
-
- Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
- channel.put(event);
-
- transaction.commit();
- transaction.close();
-
- boolean failed;
- try {
- Sink.Status status = sink.process();
- failed = false;
- } catch (EventDeliveryException ex) {
- // This is correct
- failed = true;
- }
-
- sink.stop();
- Assert.assertTrue(LifecycleController.waitForOneOf(sink,
- LifecycleState.STOP_OR_ERROR, 5000));
-
- return failed;
- }
-
-}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestAvroSource.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestAvroSource.java
deleted file mode 100644
index 7cd2dc07f..000000000
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestAvroSource.java
+++ /dev/null
@@ -1,610 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.source;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.security.cert.X509Certificate;
-import java.nio.channels.ServerSocketChannel;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.function.Consumer;
-
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLEngine;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
-
-import org.apache.avro.AvroRuntimeException;
-import org.apache.avro.ipc.netty.NettyTransceiver;
-import org.apache.avro.ipc.specific.SpecificRequestor;
-import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
-import org.apache.flume.ChannelSelector;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.FlumeException;
-import org.apache.flume.Transaction;
-import org.apache.flume.channel.ChannelProcessor;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.channel.ReplicatingChannelSelector;
-import org.apache.flume.conf.Configurables;
-import org.apache.flume.instrumentation.SourceCounter;
-import org.apache.flume.lifecycle.LifecycleController;
-import org.apache.flume.lifecycle.LifecycleState;
-import org.apache.flume.source.avro.AvroFlumeEvent;
-import org.apache.flume.source.avro.AvroSourceProtocol;
-import org.apache.flume.source.avro.Status;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.mockito.internal.util.reflection.Whitebox;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.netty.channel.ChannelPipeline;
-import io.netty.handler.codec.compression.JZlibDecoder;
-import io.netty.handler.codec.compression.JZlibEncoder;
-import io.netty.handler.codec.compression.ZlibEncoder;
-import io.netty.handler.ssl.SslHandler;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyListOf;
-import static org.mockito.Mockito.doThrow;
-
-public class TestAvroSource {
-
- private static final Logger logger = LoggerFactory
- .getLogger(TestAvroSource.class);
-
- private int selectedPort;
- private AvroSource source;
- private Channel channel;
- private InetAddress localhost;
-
- @Before
- public void setUp() throws UnknownHostException {
- localhost = InetAddress.getByName("127.0.0.1");
- source = new AvroSource();
- channel = new MemoryChannel();
-
- Configurables.configure(channel, new Context());
-
- List<Channel> channels = new ArrayList<Channel>();
- channels.add(channel);
-
- ChannelSelector rcs = new ReplicatingChannelSelector();
- rcs.setChannels(channels);
-
- source.setChannelProcessor(new ChannelProcessor(rcs));
- }
-
- @Test
- public void testLifecycle() throws InterruptedException, IOException {
-
- Context context = new Context();
-
- context.put("port", String.valueOf(selectedPort = getFreePort()));
- context.put("bind", "0.0.0.0");
-
- Configurables.configure(source, context);
-
- source.start();
-
- Assert
- .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
- source, LifecycleState.START_OR_ERROR));
- Assert.assertEquals("Server is started", LifecycleState.START,
- source.getLifecycleState());
-
- source.stop();
- Assert.assertTrue("Reached stop or error",
- LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
- Assert.assertEquals("Server is stopped", LifecycleState.STOP,
- source.getLifecycleState());
- }
-
- @Test
- public void testSourceStoppedOnFlumeExceptionIfPortUsed()
- throws InterruptedException, IOException {
- final String loopbackIPv4 = "127.0.0.1";
- final int port = 10500;
-
- // create a dummy socket bound to a known port.
- try (ServerSocketChannel dummyServerSocket = ServerSocketChannel.open()) {
- dummyServerSocket.socket().setReuseAddress(true);
- dummyServerSocket.socket().bind(new InetSocketAddress(loopbackIPv4, port));
-
- Context context = new Context();
- context.put("port", String.valueOf(port));
- context.put("bind", loopbackIPv4);
- Configurables.configure(source, context);
- try {
- source.start();
- Assert.fail("Expected an exception during startup caused by binding on a used port");
- } catch (FlumeException e) {
- logger.info("Received an expected exception.", e);
- Assert.assertTrue("Expected a server socket setup related root cause",
- e.getMessage().contains("server socket"));
- }
- }
- // As port is already in use, an exception is thrown and the source is stopped
- // cleaning up the opened sockets during source.start().
- Assert.assertEquals("Server is stopped", LifecycleState.STOP,
- source.getLifecycleState());
- }
-
- @Test
- public void testInvalidAddress()
- throws InterruptedException, IOException {
- final String invalidHost = "invalid.host";
- final int port = 10501;
-
- Context context = new Context();
- context.put("port", String.valueOf(port));
- context.put("bind", invalidHost);
- Configurables.configure(source, context);
-
- try {
- source.start();
- Assert.fail("Expected an exception during startup caused by binding on a invalid host");
- } catch (FlumeException e) {
- logger.info("Received an expected exception.", e);
- Assert.assertTrue("Expected a server socket setup related root cause",
- e.getMessage().contains("server socket"));
- }
-
- // As port is already in use, an exception is thrown and the source is stopped
- // cleaning up the opened sockets during source.start().
- Assert.assertEquals("Server is stopped", LifecycleState.STOP,
- source.getLifecycleState());
- }
-
- @Test
- public void testRequestWithNoCompression() throws InterruptedException, IOException {
-
- doRequest(false, false, 6);
- }
-
- @Test
- public void testRequestWithCompressionOnClientAndServerOnLevel0()
- throws InterruptedException, IOException {
-
- doRequest(true, true, 0);
- }
-
- @Test
- public void testRequestWithCompressionOnClientAndServerOnLevel1()
- throws InterruptedException, IOException {
-
- doRequest(true, true, 1);
- }
-
- @Test
- public void testRequestWithCompressionOnClientAndServerOnLevel6()
- throws InterruptedException, IOException {
-
- doRequest(true, true, 6);
- }
-
- @Test
- public void testRequestWithCompressionOnClientAndServerOnLevel9()
- throws InterruptedException, IOException {
-
- doRequest(true, true, 9);
- }
-
- @Test(expected = org.apache.avro.AvroRuntimeException.class)
- public void testRequestWithCompressionOnServerOnly() throws InterruptedException, IOException {
- //This will fail because both client and server need compression on
- doRequest(true, false, 6);
- }
-
- @Test(expected = org.apache.avro.AvroRuntimeException.class)
- public void testRequestWithCompressionOnClientOnly() throws InterruptedException, IOException {
- //This will fail because both client and server need compression on
- doRequest(false, true, 6);
- }
-
- private void doRequest(boolean serverEnableCompression, boolean clientEnableCompression,
- int compressionLevel) throws InterruptedException, IOException {
-
- Context context = new Context();
- context.put("port", String.valueOf(selectedPort = getFreePort()));
- context.put("bind", "0.0.0.0");
- context.put("threads", "50");
- if (serverEnableCompression) {
- context.put("compression-type", "deflate");
- } else {
- context.put("compression-type", "none");
- }
-
- Configurables.configure(source, context);
-
- source.start();
-
- Assert
- .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
- source, LifecycleState.START_OR_ERROR));
- Assert.assertEquals("Server is started", LifecycleState.START,
- source.getLifecycleState());
-
- AvroSourceProtocol client;
- NettyTransceiver nettyTransceiver;
- if (clientEnableCompression) {
-
- nettyTransceiver = new NettyTransceiver(new InetSocketAddress(selectedPort),
- (ch) -> {
- ChannelPipeline pipeline = ch.pipeline();
- ZlibEncoder encoder = new JZlibEncoder(compressionLevel);
- pipeline.addFirst("deflater", encoder);
- pipeline.addFirst("inflater", new JZlibDecoder());
- });
- client = SpecificRequestor.getClient(AvroSourceProtocol.class, nettyTransceiver);
- } else {
- nettyTransceiver = new NettyTransceiver(new InetSocketAddress(selectedPort));
- client = SpecificRequestor.getClient(AvroSourceProtocol.class, nettyTransceiver);
- }
-
- AvroFlumeEvent avroEvent = new AvroFlumeEvent();
-
- avroEvent.setHeaders(new HashMap<>());
- avroEvent.setBody(ByteBuffer.wrap("Hello avro".getBytes()));
-
- Status status = client.append(avroEvent);
-
- Assert.assertEquals(Status.OK, status);
-
- Transaction transaction = channel.getTransaction();
- transaction.begin();
-
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertEquals("Channel contained our event", "Hello avro",
- new String(event.getBody()));
- transaction.commit();
- transaction.close();
-
- logger.debug("Round trip event:{}", event);
-
-
- nettyTransceiver.close();
- source.stop();
- Assert.assertTrue("Reached stop or error",
- LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
- Assert.assertEquals("Server is stopped", LifecycleState.STOP,
- source.getLifecycleState());
- }
-
- @Test
- public void testSslRequestWithComponentKeystore() throws InterruptedException, IOException {
-
- Context context = new Context();
-
- context.put("port", String.valueOf(selectedPort = getFreePort()));
- context.put("bind", "0.0.0.0");
- context.put("ssl", "true");
- context.put("keystore", "src/test/resources/server.p12");
- context.put("keystore-password", "password");
- context.put("keystore-type", "PKCS12");
-
- Configurables.configure(source, context);
-
- doSslRequest();
- }
-
- @Test
- public void testSslRequestWithGlobalKeystore() throws InterruptedException, IOException {
-
- System.setProperty("javax.net.ssl.keyStore", "src/test/resources/server.p12");
- System.setProperty("javax.net.ssl.keyStorePassword", "password");
- System.setProperty("javax.net.ssl.keyStoreType", "PKCS12");
-
- Context context = new Context();
-
- context.put("port", String.valueOf(selectedPort = getFreePort()));
- context.put("bind", "0.0.0.0");
- context.put("ssl", "true");
-
- Configurables.configure(source, context);
-
- doSslRequest();
-
- System.clearProperty("javax.net.ssl.keyStore");
- System.clearProperty("javax.net.ssl.keyStorePassword");
- }
-
- private void doSslRequest() throws InterruptedException, IOException {
- source.start();
-
- Assert
- .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
- source, LifecycleState.START_OR_ERROR));
- Assert.assertEquals("Server is started", LifecycleState.START,
- source.getLifecycleState());
-
- AvroSourceProtocol client = SpecificRequestor.getClient(
- AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(selectedPort),
- (ch) -> {
- ChannelPipeline pipeline = ch.pipeline();
- SSLEngine engine = createSSLEngine();
- engine.setUseClientMode(true);
- pipeline.addFirst("ssl", new SslHandler(engine));
- }));
-
- AvroFlumeEvent avroEvent = new AvroFlumeEvent();
-
- avroEvent.setHeaders(new HashMap<>());
- avroEvent.setBody(ByteBuffer.wrap("Hello avro ssl".getBytes()));
-
- Status status = client.append(avroEvent);
-
- Assert.assertEquals(Status.OK, status);
-
- Transaction transaction = channel.getTransaction();
- transaction.begin();
-
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertEquals("Channel contained our event", "Hello avro ssl",
- new String(event.getBody()));
- transaction.commit();
- transaction.close();
-
- logger.debug("Round trip event:{}", event);
-
- source.stop();
- Assert.assertTrue("Reached stop or error",
- LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
- Assert.assertEquals("Server is stopped", LifecycleState.STOP,
- source.getLifecycleState());
- }
-
- private SSLEngine createSSLEngine() {
- try {
- SSLContext sslContext = SSLContext.getInstance("TLS");
- sslContext.init(null, new TrustManager[]{new PermissiveTrustManager()}, null);
- SSLEngine sslEngine = sslContext.createSSLEngine();
- sslEngine.setUseClientMode(true);
- return sslEngine;
- } catch (Exception ex) {
- throw new RuntimeException("Cannot create SSL channel", ex);
- }
- }
-
- /**
- * Bogus trust manager accepting any certificate
- */
- private static class PermissiveTrustManager implements X509TrustManager {
- @Override
- public void checkClientTrusted(X509Certificate[] certs, String s) {
- // nothing
- }
-
- @Override
- public void checkServerTrusted(X509Certificate[] certs, String s) {
- // nothing
- }
-
- @Override
- public X509Certificate[] getAcceptedIssuers() {
- return new X509Certificate[0];
- }
- }
-
- @Test
- public void testValidIpFilterAllows()
- throws InterruptedException, IOException {
- doIpFilterTest(localhost, "allow:name:localhost,deny:ip:*", true, false);
- doIpFilterTest(localhost, "allow:ip:" + localhost.getHostAddress() +
- ",deny:ip:*", true, false);
- doIpFilterTest(localhost, "allow:ip:*", true, false);
- doIpFilterTest(localhost, "allow:ip:" +
- localhost.getHostAddress().substring(0, 3) +
- "*,deny:ip:*", true, false);
- doIpFilterTest(localhost, "allow:ip:127.0.0.2,allow:ip:" +
- localhost.getHostAddress().substring(0, 3) +
- "*,deny:ip:*", true, false);
- doIpFilterTest(localhost, "allow:name:localhost,deny:ip:*", true, true);
- doIpFilterTest(localhost, "allow:ip:*", true, true);
- }
-
- @Test
- public void testValidIpFilterDenys()
- throws InterruptedException, IOException {
- doIpFilterTest(localhost, "deny:ip:*", false, false);
- doIpFilterTest(localhost, "deny:name:localhost", false, false);
- doIpFilterTest(localhost, "deny:ip:" + localhost.getHostAddress() +
- ",allow:ip:*", false, false);
- doIpFilterTest(localhost, "deny:ip:*", false, false);
- doIpFilterTest(localhost, "allow:ip:45.2.2.2,deny:ip:*", false, false);
- doIpFilterTest(localhost, "deny:ip:" +
- localhost.getHostAddress().substring(0, 3) +
- "*,allow:ip:*", false, false);
- doIpFilterTest(localhost, "deny:ip:*", false, true);
- }
-
- @Test
- public void testInvalidIpFilter() throws InterruptedException, IOException {
- doIpFilterTest(localhost, "deny:ip:*", false, false);
- doIpFilterTest(localhost, "allow:name:localhost", true, false);
- doIpFilterTest(localhost, "deny:ip:127.0.0.2,allow:ip:*,deny:ip:" +
- localhost.getHostAddress(), true, false);
- doIpFilterTest(localhost, "deny:ip:" +
- localhost.getHostAddress().substring(0, 3) + "*,allow:ip:*",
- false, false);
-
- // Private lambda expression to check the received FlumeException within this test
- Consumer<Exception> exceptionChecker = (Exception ex) -> {
- logger.info("Received an expected exception", ex);
- //covers all ipFilter related exceptions
- Assert.assertTrue("Expected an ipFilterRules related exception",
- ex.getMessage().contains("ipFilter"));
- };
-
- try {
- doIpFilterTest(localhost, null, false, false);
- Assert.fail("The null ipFilterRules config should have thrown an exception.");
- } catch (FlumeException e) {
- exceptionChecker.accept(e);
- }
-
- try {
- doIpFilterTest(localhost, "", true, false);
- Assert.fail("The empty string ipFilterRules config should have thrown "
- + "an exception");
- } catch (FlumeException e) {
- exceptionChecker.accept(e);
- }
-
- try {
- doIpFilterTest(localhost, "homer:ip:45.4.23.1", true, false);
- Assert.fail("Bad ipFilterRules config should have thrown an exception.");
- } catch (FlumeException e) {
- exceptionChecker.accept(e);
- }
-
- try {
- doIpFilterTest(localhost, "allow:sleeps:45.4.23.1", true, false);
- Assert.fail("Bad ipFilterRules config should have thrown an exception.");
- } catch (FlumeException e) {
- exceptionChecker.accept(e);
- }
- }
-
- public void doIpFilterTest(InetAddress dest, String ruleDefinition,
- boolean eventShouldBeAllowed, boolean testWithSSL)
- throws InterruptedException, IOException {
- Context context = new Context();
- context.put("port", String.valueOf(selectedPort = getFreePort()));
- context.put("bind", "0.0.0.0");
- context.put("ipFilter", "true");
- if (ruleDefinition != null) {
- context.put("ipFilterRules", ruleDefinition);
- }
- if (testWithSSL) {
- logger.info("Client testWithSSL" + testWithSSL);
- context.put("ssl", "true");
- context.put("keystore", "src/test/resources/server.p12");
- context.put("keystore-password", "password");
- context.put("keystore-type", "PKCS12");
- }
- // Invalid configuration may result in a FlumeException
- Configurables.configure(source, context);
- source.start();
-
- Assert
- .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
- source, LifecycleState.START_OR_ERROR));
- Assert.assertEquals("Server is started", LifecycleState.START,
- source.getLifecycleState());
-
- AvroSourceProtocol client;
- NettyTransceiver nettyTransceiver = null;
- try {
- if (testWithSSL) {
- nettyTransceiver = new NettyTransceiver(new InetSocketAddress(dest, selectedPort),
- (ch) -> {
- ChannelPipeline pipeline = ch.pipeline();
- SSLEngine engine = createSSLEngine();
- if (engine != null) {
- engine.setUseClientMode(true);
- pipeline.addFirst("ssl", new SslHandler(engine));
- }
- });
- client = SpecificRequestor.getClient(AvroSourceProtocol.class, nettyTransceiver);
- } else {
- nettyTransceiver = new NettyTransceiver(new InetSocketAddress(dest, selectedPort));
- client = SpecificRequestor.getClient(AvroSourceProtocol.class, nettyTransceiver);
- }
-
- AvroFlumeEvent avroEvent = new AvroFlumeEvent();
- avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
- avroEvent.setBody(ByteBuffer.wrap("Hello avro ipFilter".getBytes()));
-
- logger.info("Client about to append");
- Status status = client.append(avroEvent);
- logger.info("Client appended");
- Assert.assertEquals(Status.OK, status);
- } catch (IOException | AvroRuntimeException e) {
- Assert.assertTrue("Should have been allowed: " + ruleDefinition, !eventShouldBeAllowed);
- return;
- } finally {
- if (nettyTransceiver != null) {
- nettyTransceiver.close();
- }
- source.stop();
- }
- Assert.assertTrue("Should have been denied: " + ruleDefinition,
- eventShouldBeAllowed);
-
- Transaction transaction = channel.getTransaction();
- transaction.begin();
-
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertEquals("Channel contained our event", "Hello avro ipFilter",
- new String(event.getBody()));
- transaction.commit();
- transaction.close();
- logger.debug("Round trip event:{}", event);
-
- Assert.assertTrue("Reached stop or error",
- LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
- Assert.assertEquals("Server is stopped", LifecycleState.STOP,
- source.getLifecycleState());
- }
-
- @Test
- public void testErrorCounterChannelWriteFail() throws Exception {
- Context context = new Context();
- context.put("port", String.valueOf(selectedPort = getFreePort()));
- context.put("bind", "0.0.0.0");
- source.configure(context);
- ChannelProcessor cp = Mockito.mock(ChannelProcessor.class);
- doThrow(new ChannelException("dummy")).when(cp).processEvent(any(Event.class));
- doThrow(new ChannelException("dummy")).when(cp).processEventBatch(anyListOf(Event.class));
- source.setChannelProcessor(cp);
- source.start();
- AvroFlumeEvent avroEvent = new AvroFlumeEvent();
- avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
- avroEvent.setBody(ByteBuffer.wrap("Hello avro ssl".getBytes()));
- source.append(avroEvent);
- source.appendBatch(Arrays.asList(avroEvent));
- SourceCounter sc = (SourceCounter) Whitebox.getInternalState(source, "sourceCounter");
- Assert.assertEquals(2, sc.getChannelWriteFail());
- source.stop();
- }
-
- private static int getFreePort() throws IOException {
- try (ServerSocket socket = new ServerSocket(0)) {
- return socket.getLocalPort();
- }
- }
-
-
-}
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestNetcatSource.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestNetcatSource.java
index 7db8983fe..aa2c023c6 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestNetcatSource.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/source/TestNetcatSource.java
@@ -53,7 +53,7 @@ import java.util.List;
public class TestNetcatSource {
private static final Logger logger =
- LoggerFactory.getLogger(TestAvroSource.class);
+ LoggerFactory.getLogger(TestNetcatSource.class);
private static int getFreePort() {
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestThriftSource.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestThriftSource.java
deleted file mode 100644
index 28be4a1e1..000000000
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestThriftSource.java
+++ /dev/null
@@ -1,383 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.source;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
-import org.apache.flume.ChannelSelector;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.FlumeException;
-import org.apache.flume.Transaction;
-import org.apache.flume.api.RpcClient;
-import org.apache.flume.api.RpcClientConfigurationConstants;
-import org.apache.flume.api.RpcClientFactory;
-import org.apache.flume.channel.ChannelProcessor;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.channel.ReplicatingChannelSelector;
-import org.apache.flume.conf.Configurables;
-import org.apache.flume.event.EventBuilder;
-
-import org.apache.flume.instrumentation.SourceCounter;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.Ignore;
-import org.mockito.Mockito;
-import org.mockito.internal.util.reflection.Whitebox;
-
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyListOf;
-import static org.mockito.Mockito.doThrow;
-
-public class TestThriftSource {
-
- private ThriftSource source;
- private MemoryChannel channel;
- private RpcClient client;
- private final Properties props = new Properties();
- private int port;
-
- @Before
- public void setUp() throws IOException {
- try (ServerSocket socket = new ServerSocket(0)) {
- port = socket.getLocalPort();
- }
- props.clear();
- props.setProperty("hosts", "h1");
- props.setProperty("hosts.h1", "0.0.0.0:" + String.valueOf(port));
- props.setProperty(RpcClientConfigurationConstants.CONFIG_BATCH_SIZE, "10");
- props.setProperty(RpcClientConfigurationConstants.CONFIG_REQUEST_TIMEOUT, "2000");
- channel = new MemoryChannel();
- source = new ThriftSource();
- }
-
- @After
- public void stop() throws Exception {
- source.stop();
- }
-
- private void configureSource() {
- List<Channel> channels = new ArrayList<Channel>();
- channels.add(channel);
-
- ChannelSelector rcs = new ReplicatingChannelSelector();
- rcs.setChannels(channels);
-
- source.setChannelProcessor(new ChannelProcessor(rcs));
- }
-
- @Ignore("This test is flakey and causes tests to fail pretty often.")
- @Test
- public void testAppendSSLWithComponentKeystore() throws Exception {
-
- Context context = new Context();
- channel.configure(context);
- configureSource();
-
- context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
- context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
- context.put("ssl", "true");
- context.put("keystore", "src/test/resources/keystorefile.jks");
- context.put("keystore-password", "password");
- context.put("keystore-type", "JKS");
-
- Configurables.configure(source, context);
-
- doAppendSSL();
- }
-
- @Ignore("This test is flakey and causes tests to fail pretty often.")
- @Test
- public void testAppendSSLWithGlobalKeystore() throws Exception {
-
- System.setProperty("javax.net.ssl.keyStore", "src/test/resources/keystorefile.jks");
- System.setProperty("javax.net.ssl.keyStorePassword", "password");
- System.setProperty("javax.net.ssl.keyStoreType", "JKS");
-
- Context context = new Context();
- channel.configure(context);
- configureSource();
-
- context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
- context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
- context.put("ssl", "true");
-
- Configurables.configure(source, context);
-
- doAppendSSL();
-
- System.clearProperty("javax.net.ssl.keyStore");
- System.clearProperty("javax.net.ssl.keyStorePassword");
- System.clearProperty("javax.net.ssl.keyStoreType");
- }
-
- private void doAppendSSL() throws EventDeliveryException {
- Properties sslprops = (Properties)props.clone();
- sslprops.put("ssl", "true");
- sslprops.put("truststore", "src/test/resources/truststorefile.jks");
- sslprops.put("truststore-password", "password");
- client = RpcClientFactory.getThriftInstance(sslprops);
-
- source.start();
- for (int i = 0; i < 30; i++) {
- client.append(EventBuilder.withBody(String.valueOf(i).getBytes()));
- }
- Transaction transaction = channel.getTransaction();
- transaction.begin();
-
- for (int i = 0; i < 30; i++) {
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertEquals(String.valueOf(i), new String(event.getBody()));
- }
- transaction.commit();
- transaction.close();
- }
-
- @Test
- public void testAppend() throws Exception {
- client = RpcClientFactory.getThriftInstance(props);
- Context context = new Context();
- channel.configure(context);
- configureSource();
- context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
- context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
- Configurables.configure(source, context);
- source.start();
- for (int i = 0; i < 30; i++) {
- client.append(EventBuilder.withBody(String.valueOf(i).getBytes()));
- }
- Transaction transaction = channel.getTransaction();
- transaction.begin();
-
- for (int i = 0; i < 30; i++) {
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertEquals(String.valueOf(i), new String(event.getBody()));
- }
- transaction.commit();
- transaction.close();
- }
-
- @Test
- public void testAppendBatch() throws Exception {
- client = RpcClientFactory.getThriftInstance(props);
- Context context = new Context();
- context.put("capacity", "1000");
- context.put("transactionCapacity", "1000");
- channel.configure(context);
- configureSource();
- context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
- context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
- Configurables.configure(source, context);
- source.start();
- for (int i = 0; i < 30; i++) {
- List<Event> events = Lists.newArrayList();
- for (int j = 0; j < 10; j++) {
- Map<String, String> hdrs = Maps.newHashMap();
- hdrs.put("time", String.valueOf(System.currentTimeMillis()));
- events.add(EventBuilder.withBody(String.valueOf(i).getBytes(), hdrs));
- }
- client.appendBatch(events);
- }
- Transaction transaction = channel.getTransaction();
- transaction.begin();
- long after = System.currentTimeMillis();
- List<Integer> events = Lists.newArrayList();
- for (int i = 0; i < 300; i++) {
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertTrue(Long.valueOf(event.getHeaders().get("time")) <= after);
- events.add(Integer.parseInt(new String(event.getBody())));
- }
- transaction.commit();
- transaction.close();
-
- Collections.sort(events);
-
- int index = 0;
- //30 batches of 10
- for (int i = 0; i < 30; i++) {
- for (int j = 0; j < 10; j++) {
- Assert.assertEquals(i, events.get(index++).intValue());
- }
- }
- }
-
- @Test
- public void testAppendBigBatch() throws Exception {
- client = RpcClientFactory.getThriftInstance(props);
- Context context = new Context();
- context.put("capacity", "3000");
- context.put("transactionCapacity", "3000");
- channel.configure(context);
- configureSource();
- context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
- context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
- Configurables.configure(source, context);
- source.start();
- for (int i = 0; i < 5; i++) {
- List<Event> events = Lists.newArrayList();
- for (int j = 0; j < 500; j++) {
- Map<String, String> hdrs = Maps.newHashMap();
- hdrs.put("time", String.valueOf(System.currentTimeMillis()));
- events.add(EventBuilder.withBody(String.valueOf(i).getBytes(), hdrs));
- }
- client.appendBatch(events);
- }
- Transaction transaction = channel.getTransaction();
- transaction.begin();
- long after = System.currentTimeMillis();
- List<Integer> events = Lists.newArrayList();
- for (int i = 0; i < 2500; i++) {
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertTrue(Long.valueOf(event.getHeaders().get("time")) < after);
- events.add(Integer.parseInt(new String(event.getBody())));
- }
- transaction.commit();
- transaction.close();
-
- Collections.sort(events);
-
- int index = 0;
- //10 batches of 500
- for (int i = 0; i < 5; i++) {
- for (int j = 0; j < 500; j++) {
- Assert.assertEquals(i, events.get(index++).intValue());
- }
- }
- }
-
- @Test
- public void testMultipleClients() throws Exception {
- ExecutorService submitter = Executors.newCachedThreadPool();
- client = RpcClientFactory.getThriftInstance(props);
- Context context = new Context();
- context.put("capacity", "1000");
- context.put("transactionCapacity", "1000");
- channel.configure(context);
- configureSource();
- context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
- context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
- Configurables.configure(source, context);
- source.start();
- ExecutorCompletionService<Void> completionService = new ExecutorCompletionService<>(submitter);
- for (int i = 0; i < 30; i++) {
- completionService.submit(new SubmitHelper(i), null);
- }
- //wait for all threads to be done
-
-
- for (int i = 0; i < 30; i++) {
- completionService.take();
- }
-
- Transaction transaction = channel.getTransaction();
- transaction.begin();
- long after = System.currentTimeMillis();
- List<Integer> events = Lists.newArrayList();
- for (int i = 0; i < 300; i++) {
- Event event = channel.take();
- Assert.assertNotNull(event);
- Assert.assertTrue(Long.valueOf(event.getHeaders().get("time")) < after);
- events.add(Integer.parseInt(new String(event.getBody())));
- }
- transaction.commit();
- transaction.close();
-
- Collections.sort(events);
-
- int index = 0;
- //30 batches of 10
- for (int i = 0; i < 30; i++) {
- for (int j = 0; j < 10; j++) {
- Assert.assertEquals(i, events.get(index++).intValue());
- }
- }
- }
-
- @Test
- public void testErrorCounterChannelWriteFail() throws Exception {
- client = RpcClientFactory.getThriftInstance(props);
- Context context = new Context();
- context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
- context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
- source.configure(context);
- ChannelProcessor cp = Mockito.mock(ChannelProcessor.class);
- doThrow(new ChannelException("dummy")).when(cp).processEvent(any(Event.class));
- doThrow(new ChannelException("dummy")).when(cp).processEventBatch(anyListOf(Event.class));
- source.setChannelProcessor(cp);
- source.start();
- Event event = EventBuilder.withBody("hello".getBytes());
- try {
- client.append(event);
- } catch (EventDeliveryException e) {
- //
- }
- try {
- client.appendBatch(Arrays.asList(event));
- } catch (EventDeliveryException e) {
- //
- }
- SourceCounter sc = (SourceCounter) Whitebox.getInternalState(source, "sourceCounter");
- Assert.assertEquals(2, sc.getChannelWriteFail());
- source.stop();
- }
-
- private class SubmitHelper implements Runnable {
- private final int i;
-
- public SubmitHelper(int i) {
- this.i = i;
- }
-
- @Override
- public void run() {
- List<Event> events = Lists.newArrayList();
- for (int j = 0; j < 10; j++) {
- Map<String, String> hdrs = Maps.newHashMap();
- hdrs.put("time", String.valueOf(System.currentTimeMillis()));
- events.add(EventBuilder.withBody(String.valueOf(i).getBytes(), hdrs));
- }
- try {
- client.appendBatch(events);
- } catch (EventDeliveryException e) {
- throw new FlumeException(e);
- }
- }
- }
-}
diff --git a/flume-ng-core/src/test/resources/TestResettableFileInputStream_1.avro b/flume-ng-core/src/test/resources/TestResettableFileInputStream_1.avro
deleted file mode 100644
index 004d463b2..000000000
Binary files a/flume-ng-core/src/test/resources/TestResettableFileInputStream_1.avro and /dev/null differ
diff --git a/flume-ng-core/src/test/resources/TestResettableFileInputStream_1.truncated.avro b/flume-ng-core/src/test/resources/TestResettableFileInputStream_1.truncated.avro
deleted file mode 100644
index 31b217baf..000000000
Binary files a/flume-ng-core/src/test/resources/TestResettableFileInputStream_1.truncated.avro and /dev/null differ
diff --git a/flume-ng-core/src/test/resources/syslog_event.avsc b/flume-ng-core/src/test/resources/syslog_event.avsc
deleted file mode 100644
index 6b0d0a2fa..000000000
--- a/flume-ng-core/src/test/resources/syslog_event.avsc
+++ /dev/null
@@ -1,9 +0,0 @@
-{ "type": "record", "name": "SyslogEvent", "namespace": "org.apache.flume.test",
- "fields": [
- { "name": "facility", "type": "int" },
- { "name": "severity", "type": "int" },
- { "name": "timestamp", "type": "long" },
- { "name": "hostname", "type": "string" },
- { "name": "message", "type": "string" }
- ]
-}
diff --git a/flume-ng-sdk/pom.xml b/flume-ng-sdk/pom.xml
deleted file mode 100644
index 1fe90ed22..000000000
--- a/flume-ng-sdk/pom.xml
+++ /dev/null
@@ -1,262 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>flume-parent</artifactId>
- <groupId>org.apache.flume</groupId>
- <version>2.0.0-SNAPSHOT</version>
- <relativePath>../flume-parent/pom.xml</relativePath>
- </parent>
-
- <artifactId>flume-ng-sdk</artifactId>
- <name>Flume NG SDK</name>
- <description>Flume Software Development Kit: Stable public API for integration with Flume 1.x</description>
-
- <properties>
- <!-- TODO fix spotbugs/pmd violations -->
- <spotbugs.maxAllowedViolations>69</spotbugs.maxAllowedViolations>
- <pmd.maxAllowedViolations>170</pmd.maxAllowedViolations>
- <module.name>org.apache.flume.sdk</module.name>
- </properties>
-
- <profiles>
- <profile>
- <id>compileThrift</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>compileThrift</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <echo file="target/compile-thrift.sh">
- LICENSE=src/main/thrift/aslv2
- THRIFT_DIR=src/main/thrift
- JAVA_DIR=target/generated-sources/thrift
- mkdir -p $JAVA_DIR 2> /dev/null
- JSTATUS=$?
- if [ $JSTATUS -ne 0 ] ; then
- echo "Could not create $JAVA_DIR. Will not generate thrift files."
- exit $JSTATUS
- fi
- for THRIFT_FILE in `ls $THRIFT_DIR/*.thrift 2> /dev/null`
- do
- thrift --gen java:hashcode -o $JAVA_DIR $THRIFT_FILE
- done
- SRC_DIR=$JAVA_DIR/gen-java/org/apache/flume/thrift/
- DEST_DIR=src/main/java/org/apache/flume/thrift
- if [ ! -d $DEST_DIR ] ; then
- mkdir $DEST_DIR 2> /dev/null
- STATUS=$?
- if [ $STATUS -ne 0 ] ; then
- echo "Could not create $DEST_DIR. Will not generate thrift files."
- exit $STATUS
- fi
- fi
- for JAVA_FILE in `ls $SRC_DIR/*.java 2> /dev/null`
- do
- echo $JAVA_FILE
- cat $LICENSE > $JAVA_FILE.tmp
- cat $JAVA_FILE >> $JAVA_FILE.tmp
- mv $JAVA_FILE.tmp $JAVA_FILE
- cp $JAVA_FILE $DEST_DIR
- done
- rm -rf $JAVA_DIR
- </echo>
- <exec executable="sh" dir="${basedir}" failonerror="true">
- <arg line="target/compile-thrift.sh"/>
- </exec>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
- <build>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.eclipse.m2e</groupId>
- <artifactId>lifecycle-mapping</artifactId>
- <version>${lifecycle-mapping.version}</version>
- <configuration>
- <lifecycleMappingMetadata>
- <pluginExecutions>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-maven-plugin</artifactId>
- <versionRange>[${avro.version},)</versionRange>
- <goals>
- <goal>idl-protocol</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <execute />
- </action>
- </pluginExecution>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer-maven-plugin</artifactId>
- <versionRange>[${mvn-paranamer-plugin.version},)</versionRange>
- <goals>
- <goal>generate</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <ignore />
- </action>
- </pluginExecution>
- </pluginExecutions>
- </lifecycleMappingMetadata>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
- <plugins>
-
- <plugin>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-maven-plugin</artifactId>
- <configuration>
- <stringType>CharSequence</stringType>
- </configuration>
- <executions>
- <execution>
- <phase>generate-sources</phase>
- <goals>
- <goal>idl-protocol</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>run</id>
- <configuration>
- <sourceDirectory>${project.build.directory}/generated-sources/avro</sourceDirectory>
- <outputDirectory>${project.build.directory}/classes</outputDirectory>
- </configuration>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
-
- <dependencies>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <!-- only used for tests -->
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-jul</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-ipc-netty</artifactId>
- </dependency>
-
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.jcraft</groupId>
- <artifactId>jzlib</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
-
- </dependencies>
-</project>
diff --git a/flume-ng-sdk/src/main/avro/flume.avdl b/flume-ng-sdk/src/main/avro/flume.avdl
deleted file mode 100644
index 40da3ef46..000000000
--- a/flume-ng-sdk/src/main/avro/flume.avdl
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-@namespace("org.apache.flume.source.avro")
-
-protocol AvroSourceProtocol {
-
- enum Status {
- OK, FAILED, UNKNOWN
- }
-
- record AvroFlumeEvent {
- map<string> headers;
- bytes body;
- }
-
- Status append( AvroFlumeEvent event );
-
- Status appendBatch( array<AvroFlumeEvent> events );
-
-}
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/api/AbstractRpcClient.java b/flume-ng-sdk/src/main/java/org/apache/flume/api/AbstractRpcClient.java
deleted file mode 100644
index ac6646b42..000000000
--- a/flume-ng-sdk/src/main/java/org/apache/flume/api/AbstractRpcClient.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.api;
-
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.FlumeException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class AbstractRpcClient implements RpcClient {
- private static Logger logger = LoggerFactory.getLogger(AbstractRpcClient.class);
-
- protected int batchSize =
- RpcClientConfigurationConstants.DEFAULT_BATCH_SIZE;
- protected long connectTimeout =
- RpcClientConfigurationConstants.DEFAULT_CONNECT_TIMEOUT_MILLIS;
- protected long requestTimeout =
- RpcClientConfigurationConstants.DEFAULT_REQUEST_TIMEOUT_MILLIS;
-
- @Override
- public int getBatchSize() {
- return batchSize;
- }
- @Override
- public abstract void append(Event event) throws EventDeliveryException;
-
- @Override
- public abstract void appendBatch(List<Event> events)
- throws EventDeliveryException;
-
- @Override
- public abstract boolean isActive();
-
- @Override
- public abstract void close() throws FlumeException;
-
-
- /**
- * Configure the client using the given properties object.
- * @param properties
- * @throws FlumeException if the client can not be configured using this
- * method, or if the client was already configured once.
- */
- protected abstract void configure(Properties properties)
- throws FlumeException;
-
- /**
- * This is to parse the batch size config for rpc clients
- * @param properties config
- * @return batch size
- */
- public static int parseBatchSize(Properties properties) {
- String strBatchSize = properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_BATCH_SIZE);
- logger.debug("Batch size string = " + strBatchSize);
- int batchSize = RpcClientConfigurationConstants.DEFAULT_BATCH_SIZE;
- if (strBatchSize != null && !strBatchSize.isEmpty()) {
- try {
- int parsedBatch = Integer.parseInt(strBatchSize);
- if (parsedBatch < 1) {
- logger.warn("Invalid value for batchSize: {}; Using default value.", parsedBatch);
- } else {
- batchSize = parsedBatch;
- }
- } catch (NumberFormatException e) {
- logger.warn("BatchSize is not valid for RpcClient: " + strBatchSize +
- ". Default value assigned.", e);
- }
- }
-
- return batchSize;
- }
-
-}
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/api/FailoverRpcClient.java b/flume-ng-sdk/src/main/java/org/apache/flume/api/FailoverRpcClient.java
deleted file mode 100644
index b6d89d366..000000000
--- a/flume-ng-sdk/src/main/java/org/apache/flume/api/FailoverRpcClient.java
+++ /dev/null
@@ -1,291 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.api;
-
-import java.net.InetSocketAddress;
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.FlumeException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Avro/Netty implementation of {@link RpcClient} which supports failover. This
- * takes a list of hostname port combinations and connects to the next available
- * (looping back to the first) host, from a given list of agents in the order
- * provided.
- *
- *
- * The properties used to build a FailoverRpcClient must have:
- * <p><tt>hosts</tt> = <i>alias_for_host1</i> <i>alias_for_host2</i></p> ...
- * <p><tt>hosts.alias_for_host1</tt> = <i>hostname1:port1</i>. </p>
- * <p><tt>hosts.alias_for_host2</tt> = <i>hostname2:port2</i>. </p> etc
- * <p>Optionally it can also have a <p>
- * <tt>batch-size</tt> = <i>batchSize</i>
- * <tt>max-attempts</tt> = <i>maxAttempts</i>
- *
- * Given a failure, this client will attempt to append to <i>maxAttempts</i>
- * clients in the <i>hosts</i> list immediately following the failed host
- * (looping back to the beginning of the <i>hosts</i> list.
- */
-
-public class FailoverRpcClient extends AbstractRpcClient implements RpcClient {
- private volatile RpcClient client;
- private List<HostInfo> hosts;
- private Integer maxTries;
- private int lastCheckedhost;
- private boolean isActive;
- private Properties configurationProperties;
-
- private static final Logger logger = LoggerFactory
- .getLogger(FailoverRpcClient.class);
-
- protected FailoverRpcClient() {
- lastCheckedhost = -1;
- client = null;
- }
-
- //This function has to be synchronized to establish a happens-before
- //relationship for different threads that access this object
- //since shared data structures are created here.
- private synchronized void configureHosts(Properties properties)
- throws FlumeException {
- if (isActive) {
- logger.error("This client was already configured, " +
- "cannot reconfigure.");
- throw new FlumeException("This client was already configured, " +
- "cannot reconfigure.");
- }
- hosts = HostInfo.getHostInfoList(properties);
- String tries = properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_MAX_ATTEMPTS);
- if (tries == null || tries.isEmpty()) {
- maxTries = hosts.size();
- } else {
- try {
- maxTries = Integer.parseInt(tries);
- } catch (NumberFormatException e) {
- maxTries = hosts.size();
- }
- }
-
- batchSize = parseBatchSize(properties);
-
- isActive = true;
- }
-
- /**
- * Get the maximum number of "failed" hosts the client will try to establish
- * connection to before throwing an exception. Failed = was able to set up a
- * connection, but failed / returned error when the client tried to send data,
- *
- * @return The maximum number of failed retries
- */
- protected Integer getMaxTries() {
- return maxTries;
- }
-
- private synchronized RpcClient getClient() {
- if (client == null || !this.client.isActive()) {
- client = getNextClient();
- return client;
- } else {
- return client;
- }
- }
-
- /**
- * Tries to append an event to the currently connected client. If it cannot
- * send the event, it tries to send to next available host
- *
- * @param event The event to be appended.
- *
- * @throws EventDeliveryException
- */
- @Override
- public void append(Event event) throws EventDeliveryException {
- //Why a local variable rather than just calling getClient()?
- //If we get an EventDeliveryException, we need to call close on
- //that specific client, getClient in this case, will get us
- //the next client - leaving a resource leak.
- RpcClient localClient = null;
- synchronized (this) {
- if (!isActive) {
- logger.error("Attempting to append to an already closed client.");
- throw new EventDeliveryException(
- "Attempting to append to an already closed client.");
- }
- }
- // Sit in an infinite loop and try to append!
- int tries = 0;
- while (tries < maxTries) {
- try {
- tries++;
- localClient = getClient();
- localClient.append(event);
- return;
- } catch (EventDeliveryException e) {
- // Could not send event through this client, try to pick another client.
- logger.warn("Client failed. Exception follows: ", e);
- localClient.close();
- localClient = null;
- } catch (Exception e2) {
- logger.error("Failed to send event: ", e2);
- throw new EventDeliveryException(
- "Failed to send event. Exception follows: ", e2);
- }
- }
- logger.error("Tried many times, could not send event.");
- throw new EventDeliveryException("Failed to send the event!");
- }
-
- /**
- * Tries to append a list of events to the currently connected client. If it
- * cannot send the event, it tries to send to next available host
- *
- * @param events The events to be appended.
- *
- * @throws EventDeliveryException
- */
- @Override
- public void appendBatch(List<Event> events)
- throws EventDeliveryException {
- RpcClient localClient = null;
- synchronized (this) {
- if (!isActive) {
- logger.error("Attempting to append to an already closed client.");
- throw new EventDeliveryException(
- "Attempting to append to an already closed client!");
- }
- }
- int tries = 0;
- while (tries < maxTries) {
- try {
- tries++;
- localClient = getClient();
- localClient.appendBatch(events);
- return;
- } catch (EventDeliveryException e) {
- // Could not send event through this client, try to pick another client.
- logger.warn("Client failed. Exception follows: ", e);
- localClient.close();
- localClient = null;
- } catch (Exception e1) {
- logger.error("No clients active: ", e1);
- throw new EventDeliveryException("No clients currently active. " +
- "Exception follows: ", e1);
- }
- }
- logger.error("Tried many times, could not send event.");
- throw new EventDeliveryException("Failed to send the event!");
- }
-
- // Returns false if and only if this client has been closed explicitly.
- // Should we check if any clients are active, if none are then return false?
- // This method has to be lightweight, so not checking if hosts are active.
- @Override
- public synchronized boolean isActive() {
- return isActive;
- }
-
- /**
- * Close the connection. This function is safe to call over and over.
- */
- @Override
- public synchronized void close() throws FlumeException {
- if (client != null) {
- client.close();
- isActive = false;
- }
- }
-
- /**
- * Get the last socket address this client connected to. No guarantee this
- * will be the next it will connect to. If this host is down, it will connect
- * to another host. To be used only from the unit tests!
- * @return The last socket address this client connected to
- */
- protected InetSocketAddress getLastConnectedServerAddress() {
- HostInfo hostInfo = hosts.get(lastCheckedhost);
- return new InetSocketAddress(hostInfo.getHostName(),
- hostInfo.getPortNumber());
- }
-
- private RpcClient getNextClient() throws FlumeException {
- lastCheckedhost =
- (lastCheckedhost == (hosts.size() - 1)) ? -1 : lastCheckedhost;
- RpcClient localClient = null;
- int limit = hosts.size();
-
- Properties props = new Properties();
- props.putAll(configurationProperties);
- props.put(RpcClientConfigurationConstants.CONFIG_CLIENT_TYPE,
- RpcClientConfigurationConstants.DEFAULT_CLIENT_TYPE);
- //Try to connect to all hosts again, till we find one available
- for (int count = lastCheckedhost + 1; count < limit; count++) {
- HostInfo hostInfo = hosts.get(count);
- try {
- setDefaultProperties(hostInfo, props);
- localClient = RpcClientFactory.getInstance(props);
- lastCheckedhost = count;
- return localClient;
- } catch (FlumeException e) {
- logger.info("Could not connect to " + hostInfo, e);
- continue;
- }
- }
- for (int count = 0; count <= lastCheckedhost; count++) {
- HostInfo hostInfo = hosts.get(count);
- try {
- setDefaultProperties(hostInfo, props);
- localClient = RpcClientFactory.getInstance(props);
- lastCheckedhost = count;
- return localClient;
- } catch (FlumeException e) {
- logger.info("Could not connect to " + hostInfo, e);
- continue;
- }
- }
- if (localClient == null) {
- lastCheckedhost = -1;
- logger.error("No active client found.");
- throw new FlumeException("No active client.");
- }
- // This return should never be reached!
- return localClient;
- }
-
- private void setDefaultProperties(HostInfo hostInfo, Properties props) {
- props.put(RpcClientConfigurationConstants.CONFIG_CLIENT_TYPE,
- RpcClientFactory.ClientType.DEFAULT.name());
- props.put(RpcClientConfigurationConstants.CONFIG_HOSTS,
- hostInfo.getReferenceName());
- }
-
- @Override
- public void configure(Properties properties) throws FlumeException {
- configurationProperties = new Properties();
- configurationProperties.putAll(properties);
-
- configureHosts(configurationProperties);
- }
-}
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/api/HostInfo.java b/flume-ng-sdk/src/main/java/org/apache/flume/api/HostInfo.java
deleted file mode 100644
index 53d99a260..000000000
--- a/flume-ng-sdk/src/main/java/org/apache/flume/api/HostInfo.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.api;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.flume.FlumeException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A convenience class that holds the property reference name along with the
- * hostname and port number of a specified host address. It also provides
- * a method to parse out the host list from a given properties object
- * that contains the host details.
- */
-public class HostInfo {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(HostInfo.class);
-
- private final String referenceName;
- private final String hostName;
- private final int portNumber;
-
- public HostInfo(String referenceName, String hostName, int portNumber) {
- this.referenceName = referenceName;
- this.hostName = hostName;
- this.portNumber = portNumber;
- }
-
- public String getReferenceName() {
- return referenceName;
- }
-
- public String getHostName() {
- return hostName;
- }
-
- public int getPortNumber() {
- return portNumber;
- }
-
- @Override
- public String toString() {
- return referenceName + "{" + hostName + ":" + portNumber + "}";
- }
-
- public static List<HostInfo> getHostInfoList(Properties properties) {
- List<HostInfo> hosts = new ArrayList<HostInfo>();
- String hostNames = properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_HOSTS);
- String[] hostList;
- if (hostNames != null && !hostNames.isEmpty()) {
- hostList = hostNames.split("\\s+");
- for (int i = 0; i < hostList.length; i++) {
- String hostAndPortStr = properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + hostList[i]);
- // Ignore that host if value is not there
- if (hostAndPortStr != null) {
- String[] hostAndPort = hostAndPortStr.split(":");
- if (hostAndPort.length != 2) {
- LOGGER.error("Invalid host address" + hostAndPortStr);
- throw new FlumeException("Invalid host address" + hostAndPortStr);
- }
- Integer port = null;
- try {
- port = Integer.parseInt(hostAndPort[1]);
- } catch (NumberFormatException e) {
- LOGGER.error("Invalid port number" + hostAndPortStr, e);
- throw new FlumeException("Invalid port number" + hostAndPortStr);
- }
- HostInfo info = new HostInfo(hostList[i],
- hostAndPort[0].trim(), port);
- hosts.add(info);
- }
- }
- }
-
- return hosts;
- }
-}
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/api/LoadBalancingRpcClient.java b/flume-ng-sdk/src/main/java/org/apache/flume/api/LoadBalancingRpcClient.java
deleted file mode 100644
index a025612bf..000000000
--- a/flume-ng-sdk/src/main/java/org/apache/flume/api/LoadBalancingRpcClient.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.api;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.FlumeException;
-import org.apache.flume.util.OrderSelector;
-import org.apache.flume.util.RandomOrderSelector;
-import org.apache.flume.util.RoundRobinOrderSelector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p>An implementation of RpcClient interface that uses NettyAvroRpcClient
- * instances to load-balance the requests over many different hosts. This
- * implementation supports a round-robin scheme or random scheme of doing
- * load balancing over the various hosts. To specify round-robin scheme set
- * the value of the configuration property <tt>load-balance-type</tt> to
- * <tt>round_robin</tt>. Similarly, for random scheme this value should be
- * set to <tt>random</tt>, and for a custom scheme the full class name of
- * the class that implements the <tt>HostSelector</tt> interface.
- * </p>
- * <p>
- * This implementation also performs basic failover in case the randomly
- * selected host is not available for receiving the event.
- * </p>
- */
-public class LoadBalancingRpcClient extends AbstractRpcClient {
-
- private static final Logger LOGGER = LoggerFactory
- .getLogger(LoadBalancingRpcClient.class);
-
- private List<HostInfo> hosts;
- private HostSelector selector;
- private Map<String, RpcClient> clientMap;
- private Properties configurationProperties;
- private volatile boolean isOpen = false;
-
- @Override
- public void append(Event event) throws EventDeliveryException {
- throwIfClosed();
- boolean eventSent = false;
- Iterator<HostInfo> it = selector.createHostIterator();
-
- while (it.hasNext()) {
- HostInfo host = it.next();
- try {
- RpcClient client = getClient(host);
- client.append(event);
- eventSent = true;
- break;
- } catch (Exception ex) {
- selector.informFailure(host);
- LOGGER.warn("Failed to send event to host " + host, ex);
- }
- }
-
- if (!eventSent) {
- throw new EventDeliveryException("Unable to send event to any host");
- }
- }
-
- @Override
- public void appendBatch(List<Event> events) throws EventDeliveryException {
- throwIfClosed();
- boolean batchSent = false;
- Iterator<HostInfo> it = selector.createHostIterator();
-
- while (it.hasNext()) {
- HostInfo host = it.next();
- try {
- RpcClient client = getClient(host);
- client.appendBatch(events);
- batchSent = true;
- break;
- } catch (Exception ex) {
- selector.informFailure(host);
- LOGGER.warn("Failed to send batch to host " + host, ex);
- }
- }
-
- if (!batchSent) {
- throw new EventDeliveryException("Unable to send batch to any host");
- }
- }
-
- @Override
- public boolean isActive() {
- return isOpen;
- }
-
- private void throwIfClosed() throws EventDeliveryException {
- if (!isOpen) {
- throw new EventDeliveryException("Rpc Client is closed");
- }
- }
-
- @Override
- public void close() throws FlumeException {
- isOpen = false;
- synchronized (this) {
- Iterator<String> it = clientMap.keySet().iterator();
- while (it.hasNext()) {
- String name = it.next();
- RpcClient client = clientMap.get(name);
- if (client != null) {
- try {
- client.close();
- } catch (Exception ex) {
- LOGGER.warn("Failed to close client: " + name, ex);
- }
- }
- it.remove();
- }
- }
- }
-
- @Override
- protected void configure(Properties properties) throws FlumeException {
- clientMap = new HashMap<String, RpcClient>();
- configurationProperties = new Properties();
- configurationProperties.putAll(properties);
- hosts = HostInfo.getHostInfoList(properties);
- if (hosts.size() < 2) {
- throw new FlumeException("At least two hosts are required to use the "
- + "load balancing RPC client.");
- }
-
- String lbTypeName = properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_HOST_SELECTOR,
- RpcClientConfigurationConstants.HOST_SELECTOR_ROUND_ROBIN);
-
- boolean backoff = Boolean.valueOf(properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_BACKOFF,
- String.valueOf(false)));
-
- String maxBackoffStr = properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_MAX_BACKOFF);
-
- long maxBackoff = 0;
- if (maxBackoffStr != null) {
- maxBackoff = Long.parseLong(maxBackoffStr);
- }
-
- if (lbTypeName.equalsIgnoreCase(
- RpcClientConfigurationConstants.HOST_SELECTOR_ROUND_ROBIN)) {
- selector = new RoundRobinHostSelector(backoff, maxBackoff);
- } else if (lbTypeName.equalsIgnoreCase(
- RpcClientConfigurationConstants.HOST_SELECTOR_RANDOM)) {
- selector = new RandomOrderHostSelector(backoff, maxBackoff);
- } else {
- try {
- @SuppressWarnings("unchecked")
- Class<? extends HostSelector> klass = (Class<? extends HostSelector>)
- Class.forName(lbTypeName);
-
- selector = klass.newInstance();
- } catch (Exception ex) {
- throw new FlumeException("Unable to instantiate host selector: "
- + lbTypeName, ex);
- }
- }
-
- selector.setHosts(hosts);
- batchSize = parseBatchSize(properties);
- isOpen = true;
- }
-
- private synchronized RpcClient getClient(HostInfo info)
- throws FlumeException, EventDeliveryException {
- throwIfClosed();
- String name = info.getReferenceName();
- RpcClient client = clientMap.get(name);
- if (client == null) {
- client = createClient(name);
- clientMap.put(name, client);
- } else if (!client.isActive()) {
- try {
- client.close();
- } catch (Exception ex) {
- LOGGER.warn("Failed to close client for " + info, ex);
- }
- client = createClient(name);
- clientMap.put(name, client);
- }
-
- return client;
- }
-
- private RpcClient createClient(String referenceName) throws FlumeException {
- Properties props = getClientConfigurationProperties(referenceName);
- return RpcClientFactory.getInstance(props);
- }
-
- private Properties getClientConfigurationProperties(String referenceName) {
- Properties props = new Properties();
- props.putAll(configurationProperties);
- props.put(RpcClientConfigurationConstants.CONFIG_CLIENT_TYPE,
- RpcClientFactory.ClientType.DEFAULT);
- props.put(RpcClientConfigurationConstants.CONFIG_HOSTS, referenceName);
-
- return props;
- }
-
- public interface HostSelector {
-
- void setHosts(List<HostInfo> hosts);
-
- Iterator<HostInfo> createHostIterator();
-
- void informFailure(HostInfo failedHost);
- }
-
- /**
- * A host selector that implements the round-robin host selection policy.
- */
- private static class RoundRobinHostSelector implements HostSelector {
-
- private OrderSelector<HostInfo> selector;
-
- RoundRobinHostSelector(boolean backoff, long maxBackoff) {
- selector = new RoundRobinOrderSelector<HostInfo>(backoff);
- if (maxBackoff != 0) {
- selector.setMaxTimeOut(maxBackoff);
- }
- }
-
- @Override
- public synchronized Iterator<HostInfo> createHostIterator() {
- return selector.createIterator();
- }
-
- @Override
- public synchronized void setHosts(List<HostInfo> hosts) {
- selector.setObjects(hosts);
- }
-
- public synchronized void informFailure(HostInfo failedHost) {
- selector.informFailure(failedHost);
- }
- }
-
- private static class RandomOrderHostSelector implements HostSelector {
-
- private OrderSelector<HostInfo> selector;
-
- RandomOrderHostSelector(boolean backoff, Long maxBackoff) {
- selector = new RandomOrderSelector<HostInfo>(backoff);
- if (maxBackoff != 0) {
- selector.setMaxTimeOut(maxBackoff);
- }
- }
-
- @Override
- public synchronized Iterator<HostInfo> createHostIterator() {
- return selector.createIterator();
- }
-
- @Override
- public synchronized void setHosts(List<HostInfo> hosts) {
- selector.setObjects(hosts);
- }
-
- @Override
- public void informFailure(HostInfo failedHost) {
- selector.informFailure(failedHost);
- }
- }
-
-}
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/api/NettyAvroRpcClient.java b/flume-ng-sdk/src/main/java/org/apache/flume/api/NettyAvroRpcClient.java
deleted file mode 100644
index cc4956958..000000000
--- a/flume-ng-sdk/src/main/java/org/apache/flume/api/NettyAvroRpcClient.java
+++ /dev/null
@@ -1,650 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.api;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
-import java.security.KeyStore;
-import java.security.cert.X509Certificate;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.ReentrantLock;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLEngine;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-import javax.net.ssl.X509TrustManager;
-
-import org.apache.avro.ipc.CallFuture;
-import org.apache.avro.ipc.Transceiver;
-import org.apache.avro.ipc.netty.NettyTransceiver;
-import org.apache.avro.ipc.specific.SpecificRequestor;
-import org.apache.commons.lang.StringUtils;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.FlumeException;
-import org.apache.flume.source.avro.AvroFlumeEvent;
-import org.apache.flume.source.avro.AvroSourceProtocol;
-import org.apache.flume.source.avro.Status;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.netty.channel.ChannelPipeline;
-import io.netty.handler.codec.compression.JZlibDecoder;
-import io.netty.handler.codec.compression.JZlibEncoder;
-import io.netty.handler.codec.compression.ZlibEncoder;
-import io.netty.handler.ssl.SslHandler;
-
-/**
- * Avro/Netty implementation of {@link RpcClient}.
- * The connections are intended to be opened before clients are given access so
- * that the object cannot ever be in an inconsistent when exposed to users.
- */
-public class NettyAvroRpcClient extends SSLContextAwareAbstractRpcClient {
-
- private ExecutorService callTimeoutPool;
- private final ReentrantLock stateLock = new ReentrantLock();
-
- /**
- * Guarded by {@code stateLock}
- */
- private ConnState connState;
-
- private InetSocketAddress address;
-
- private Transceiver transceiver;
- private AvroSourceProtocol.Callback avroClient;
- private static final Logger logger = LoggerFactory.getLogger(NettyAvroRpcClient.class);
- private boolean enableDeflateCompression;
- private int compressionLevel;
-
- /**
- * This constructor is intended to be called from {@link RpcClientFactory}.
- * A call to this constructor should be followed by call to configure().
- */
- protected NettyAvroRpcClient() {
- }
-
- /**
- * This method should only be invoked by the build function
- * @throws FlumeException
- */
- private void connect() throws FlumeException {
- connect(connectTimeout, TimeUnit.MILLISECONDS);
- }
-
- /**
- * Internal only, for now
- * @param timeout
- * @param tu
- * @throws FlumeException
- */
- private void connect(long timeout, TimeUnit tu) throws FlumeException {
- callTimeoutPool = Executors.newCachedThreadPool(
- new TransceiverThreadFactory("Flume Avro RPC Client Call Invoker"));
-
- try {
- transceiver = new NettyTransceiver(this.address, Math.toIntExact(tu.toMillis(timeout)),
- (ch) -> {
- ChannelPipeline pipeline = ch.pipeline();
- if (enableDeflateCompression) {
- ZlibEncoder encoder = new JZlibEncoder(compressionLevel);
- pipeline.addFirst("deflater", encoder);
- pipeline.addFirst("inflater", new JZlibDecoder());
- }
- SSLEngine engine = createSSLEngine();
- if (engine != null) {
- engine.setUseClientMode(true);
- pipeline.addLast("ssl", new SslHandler(engine));
- }
- });
- avroClient = SpecificRequestor.getClient(AvroSourceProtocol.Callback.class, transceiver);
- } catch (Throwable t) {
- if (callTimeoutPool != null) {
- callTimeoutPool.shutdownNow();
- }
- if (t instanceof IOException) {
- throw new FlumeException(this + ": RPC connection error", t);
- } else if (t instanceof FlumeException) {
- throw (FlumeException) t;
- } else if (t instanceof Error) {
- throw (Error) t;
- } else {
- throw new FlumeException(this + ": Unexpected exception", t);
- }
- }
-
- setState(ConnState.READY);
- }
-
- @Override
- public void close() throws FlumeException {
- if (callTimeoutPool != null) {
- callTimeoutPool.shutdown();
- try {
- if (!callTimeoutPool.awaitTermination(requestTimeout, TimeUnit.MILLISECONDS)) {
- callTimeoutPool.shutdownNow();
- if (!callTimeoutPool.awaitTermination(requestTimeout, TimeUnit.MILLISECONDS)) {
- logger.warn(this + ": Unable to cleanly shut down call timeout pool");
- }
- }
- } catch (InterruptedException ex) {
- logger.warn(this + ": Interrupted during close", ex);
- // re-cancel if current thread also interrupted
- callTimeoutPool.shutdownNow();
- // preserve interrupt status
- Thread.currentThread().interrupt();
- }
-
- callTimeoutPool = null;
- }
- try {
- if (transceiver != null) {
- transceiver.close();
- }
- } catch (IOException ex) {
- throw new FlumeException(this + ": Error closing transceiver.", ex);
- } finally {
- setState(ConnState.DEAD);
- }
-
- }
-
- @Override
- public String toString() {
- return "NettyAvroRpcClient { host: " + address.getHostName() + ", port: " + address.getPort() + " }";
- }
-
- @Override
- public void append(Event event) throws EventDeliveryException {
- try {
- append(event, requestTimeout, TimeUnit.MILLISECONDS);
- } catch (Throwable t) {
- // we mark as no longer active without trying to clean up resources
- // client is required to call close() to clean up resources
- setState(ConnState.DEAD);
- if (t instanceof Error) {
- throw (Error) t;
- }
- if (t instanceof TimeoutException) {
- throw new EventDeliveryException(this + ": Failed to send event. " +
- "RPC request timed out after " + requestTimeout + "ms", t);
- }
- throw new EventDeliveryException(this + ": Failed to send event", t);
- }
- }
-
- private void append(Event event, long timeout, TimeUnit tu)
- throws EventDeliveryException {
-
- assertReady();
-
- final CallFuture<Status> callFuture = new CallFuture<Status>();
-
- final AvroFlumeEvent avroEvent = new AvroFlumeEvent();
- avroEvent.setBody(ByteBuffer.wrap(event.getBody()));
- avroEvent.setHeaders(toCharSeqMap(event.getHeaders()));
-
- Future<Void> handshake;
- try {
- // due to AVRO-1122, avroClient.append() may block
- handshake = callTimeoutPool.submit(new Callable<Void>() {
-
- @Override
- public Void call() throws Exception {
- avroClient.append(avroEvent, callFuture);
- return null;
- }
- });
- } catch (RejectedExecutionException ex) {
- throw new EventDeliveryException(this + ": Executor error", ex);
- }
-
- try {
- handshake.get(connectTimeout, TimeUnit.MILLISECONDS);
- } catch (TimeoutException ex) {
- throw new EventDeliveryException(this + ": Handshake timed out after " + connectTimeout + " ms", ex);
- } catch (InterruptedException ex) {
- throw new EventDeliveryException(this + ": Interrupted in handshake", ex);
- } catch (ExecutionException ex) {
- throw new EventDeliveryException(this + ": RPC request exception", ex);
- } catch (CancellationException ex) {
- throw new EventDeliveryException(this + ": RPC request cancelled", ex);
- } finally {
- if (!handshake.isDone()) {
- handshake.cancel(true);
- }
- }
-
- waitForStatusOK(callFuture, timeout, tu);
- }
-
- @Override
- public void appendBatch(List<Event> events) throws EventDeliveryException {
- try {
- appendBatch(events, requestTimeout, TimeUnit.MILLISECONDS);
- } catch (Throwable t) {
- // we mark as no longer active without trying to clean up resources
- // client is required to call close() to clean up resources
- setState(ConnState.DEAD);
- if (t instanceof Error) {
- throw (Error) t;
- }
- if (t instanceof TimeoutException) {
- throw new EventDeliveryException(this + ": Failed to send event. " +
- "RPC request timed out after " + requestTimeout + " ms", t);
- }
- throw new EventDeliveryException(this + ": Failed to send batch", t);
- }
- }
-
- private void appendBatch(List<Event> events, long timeout, TimeUnit tu)
- throws EventDeliveryException {
-
- assertReady();
-
- Iterator<Event> iter = events.iterator();
- final List<AvroFlumeEvent> avroEvents = new LinkedList<AvroFlumeEvent>();
-
- // send multiple batches... bail if there is a problem at any time
- while (iter.hasNext()) {
- avroEvents.clear();
-
- for (int i = 0; i < batchSize && iter.hasNext(); i++) {
- Event event = iter.next();
- AvroFlumeEvent avroEvent = new AvroFlumeEvent();
- avroEvent.setBody(ByteBuffer.wrap(event.getBody()));
- avroEvent.setHeaders(toCharSeqMap(event.getHeaders()));
- avroEvents.add(avroEvent);
- }
-
- final CallFuture<Status> callFuture = new CallFuture<Status>();
-
- Future<Void> handshake;
- try {
- // due to AVRO-1122, avroClient.appendBatch() may block
- handshake = callTimeoutPool.submit(new Callable<Void>() {
-
- @Override
- public Void call() throws Exception {
- avroClient.appendBatch(avroEvents, callFuture);
- return null;
- }
- });
- } catch (RejectedExecutionException ex) {
- throw new EventDeliveryException(this + ": Executor error", ex);
- }
-
- try {
- handshake.get(connectTimeout, TimeUnit.MILLISECONDS);
- } catch (TimeoutException ex) {
- throw new EventDeliveryException(this + ": Handshake timed out after " +
- connectTimeout + "ms", ex);
- } catch (InterruptedException ex) {
- throw new EventDeliveryException(this + ": Interrupted in handshake",
- ex);
- } catch (ExecutionException ex) {
- throw new EventDeliveryException(this + ": RPC request exception", ex);
- } catch (CancellationException ex) {
- throw new EventDeliveryException(this + ": RPC request cancelled", ex);
- } finally {
- if (!handshake.isDone()) {
- handshake.cancel(true);
- }
- }
-
- waitForStatusOK(callFuture, timeout, tu);
- }
- }
-
- /**
- * Helper method that waits for a Status future to come back and validates
- * that it returns Status == OK.
- * @param callFuture Future to wait on
- * @param timeout Time to wait before failing
- * @param tu Time Unit of {@code timeout}
- * @throws EventDeliveryException If there is a timeout or if Status != OK
- */
- private void waitForStatusOK(CallFuture<Status> callFuture,
- long timeout, TimeUnit tu) throws EventDeliveryException {
- try {
- Status status = callFuture.get(timeout, tu);
- if (status != Status.OK) {
- throw new EventDeliveryException(this + ": Avro RPC call returned Status: " + status);
- }
- } catch (CancellationException ex) {
- throw new EventDeliveryException(this + ": RPC future was cancelled", ex);
- } catch (ExecutionException ex) {
- throw new EventDeliveryException(this + ": Exception thrown from remote handler", ex);
- } catch (TimeoutException ex) {
- throw new EventDeliveryException(this + ": RPC request timed out", ex);
- } catch (InterruptedException ex) {
- Thread.currentThread().interrupt();
- throw new EventDeliveryException(this + ": RPC request interrupted", ex);
- }
- }
-
- /**
- * This method should always be used to change {@code connState} so we ensure
- * that invalid state transitions do not occur and that the {@code isIdle}
- * {@link Condition} variable gets signaled reliably.
- * Throws {@code IllegalStateException} when called to transition from CLOSED
- * to another state.
- * @param newState
- */
- private void setState(ConnState newState) {
- stateLock.lock();
- try {
- if (connState == ConnState.DEAD && connState != newState) {
- throw new IllegalStateException("Cannot transition from CLOSED state.");
- }
- connState = newState;
- } finally {
- stateLock.unlock();
- }
- }
-
- /**
- * If the connection state != READY, throws {@link EventDeliveryException}.
- */
- private void assertReady() throws EventDeliveryException {
- stateLock.lock();
- try {
- ConnState curState = connState;
- if (curState != ConnState.READY) {
- throw new EventDeliveryException("RPC failed, client in an invalid state: " + curState);
- }
- } finally {
- stateLock.unlock();
- }
- }
-
- /**
- * Helper function to convert a map of String to a map of CharSequence.
- */
- private static Map<CharSequence, CharSequence> toCharSeqMap(Map<String, String> stringMap) {
- Map<CharSequence, CharSequence> charSeqMap = new HashMap<>();
- for (Map.Entry<String, String> entry : stringMap.entrySet()) {
- charSeqMap.put(entry.getKey(), entry.getValue());
- }
- return charSeqMap;
- }
-
- @Override
- public boolean isActive() {
- stateLock.lock();
- try {
- return (connState == ConnState.READY);
- } finally {
- stateLock.unlock();
- }
- }
-
- private static enum ConnState {
- INIT, READY, DEAD
- }
-
- /**
- * <p>
- * Configure the actual client using the properties.
- * <tt>properties</tt> should have at least 2 params:
- * <p><tt>hosts</tt> = <i>alias_for_host</i></p>
- * <p><tt>alias_for_host</tt> = <i>hostname:port</i>. </p>
- * Only the first host is added, rest are discarded.</p>
- * <p>Optionally it can also have a <p>
- * <tt>batch-size</tt> = <i>batchSize</i>
- * @param properties The properties to instantiate the client with.
- * @return
- */
- @Override
- public synchronized void configure(Properties properties) throws FlumeException {
- stateLock.lock();
- try {
- if (connState == ConnState.READY || connState == ConnState.DEAD) {
- throw new FlumeException("This client was already configured, cannot reconfigure.");
- }
- } finally {
- stateLock.unlock();
- }
-
- batchSize = parseBatchSize(properties);
-
- // host and port
- String hostNames = properties.getProperty(RpcClientConfigurationConstants.CONFIG_HOSTS);
- String[] hosts = null;
- if (hostNames != null && !hostNames.isEmpty()) {
- hosts = hostNames.split("\\s+");
- } else {
- throw new FlumeException("Hosts list is invalid: " + hostNames);
- }
-
- if (hosts.length > 1) {
- logger.warn("More than one hosts are specified for the default client. "
- + "Only the first host will be used and others ignored. Specified: "
- + hostNames + "; to be used: " + hosts[0]);
- }
-
- String host = properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + hosts[0]);
- if (host == null || host.isEmpty()) {
- throw new FlumeException("Host not found: " + hosts[0]);
- }
- String[] hostAndPort = host.split(":");
- if (hostAndPort.length != 2) {
- throw new FlumeException("Invalid hostname: " + hosts[0]);
- }
- Integer port = null;
- try {
- port = Integer.parseInt(hostAndPort[1]);
- } catch (NumberFormatException e) {
- throw new FlumeException("Invalid Port: " + hostAndPort[1], e);
- }
- this.address = new InetSocketAddress(hostAndPort[0], port);
-
- // connect timeout
- connectTimeout = RpcClientConfigurationConstants.DEFAULT_CONNECT_TIMEOUT_MILLIS;
- String strConnTimeout = properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_CONNECT_TIMEOUT);
- if (strConnTimeout != null && strConnTimeout.trim().length() > 0) {
- try {
- connectTimeout = Long.parseLong(strConnTimeout);
- if (connectTimeout < 1000) {
- logger.warn("Connection timeout specified less than 1s. Using default value instead.");
- connectTimeout = RpcClientConfigurationConstants.DEFAULT_CONNECT_TIMEOUT_MILLIS;
- }
- } catch (NumberFormatException ex) {
- logger.error("Invalid connect timeout specified: " + strConnTimeout);
- }
- }
-
- // request timeout
- requestTimeout = RpcClientConfigurationConstants.DEFAULT_REQUEST_TIMEOUT_MILLIS;
- String strReqTimeout = properties.getProperty(RpcClientConfigurationConstants.CONFIG_REQUEST_TIMEOUT);
- if (strReqTimeout != null && strReqTimeout.trim().length() > 0) {
- try {
- requestTimeout = Long.parseLong(strReqTimeout);
- if (requestTimeout < 1000) {
- logger.warn("Request timeout specified less than 1s. Using default value instead.");
- requestTimeout = RpcClientConfigurationConstants.DEFAULT_REQUEST_TIMEOUT_MILLIS;
- }
- } catch (NumberFormatException ex) {
- logger.error("Invalid request timeout specified: " + strReqTimeout);
- }
- }
-
- String enableCompressionStr =
- properties.getProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE);
- if (enableCompressionStr != null && enableCompressionStr.equalsIgnoreCase("deflate")) {
- this.enableDeflateCompression = true;
- String compressionLvlStr =
- properties.getProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_LEVEL);
- compressionLevel = RpcClientConfigurationConstants.DEFAULT_COMPRESSION_LEVEL;
- if (compressionLvlStr != null) {
- try {
- compressionLevel = Integer.parseInt(compressionLvlStr);
- } catch (NumberFormatException ex) {
- logger.error("Invalid compression level: " + compressionLvlStr);
- }
- }
- }
-
- configureSSL(properties);
-
- String maxIoWorkersStr = properties.getProperty(RpcClientConfigurationConstants.MAX_IO_WORKERS);
- if (!StringUtils.isEmpty(maxIoWorkersStr)) {
- logger.warn("Specifying the number of workers is no longer supported");
- }
-
- this.connect();
- }
-
- /**
- * A thread factor implementation modeled after the implementation of
- * NettyTransceiver.NettyTransceiverThreadFactory class which is
- * a private static class. The only difference between that and this
- * implementation is that this implementation marks all the threads daemon
- * which allows the termination of the VM when the non-daemon threads
- * are done.
- */
- private static class TransceiverThreadFactory implements ThreadFactory {
- private final AtomicInteger threadId = new AtomicInteger(0);
- private final String prefix;
-
- /**
- * Creates a TransceiverThreadFactory that creates threads with the
- * specified name.
- * @param prefix the name prefix to use for all threads created by this
- * ThreadFactory. A unique ID will be appended to this prefix to form the
- * final thread name.
- */
- public TransceiverThreadFactory(String prefix) {
- this.prefix = prefix;
- }
-
- @Override
- public Thread newThread(Runnable r) {
- Thread thread = new Thread(r);
- thread.setDaemon(true);
- thread.setName(prefix + " " + threadId.incrementAndGet());
- return thread;
- }
- }
-
- private SSLEngine createSSLEngine() {
- TrustManager[] managers;
- try {
- if (enableSsl) {
- if (trustAllCerts) {
- logger.warn(
- "No truststore configured, setting TrustManager to accept all server certificates");
- managers = new TrustManager[]{new PermissiveTrustManager()};
- } else {
- KeyStore keystore = null;
-
- if (truststore != null) {
- try (InputStream truststoreStream = new FileInputStream(truststore)) {
- keystore = KeyStore.getInstance(truststoreType);
- keystore.load(truststoreStream, truststorePassword != null ? truststorePassword.toCharArray() : null);
- }
- }
-
- TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
- // null keystore is OK, with SunX509 it defaults to system CA Certs
- // see http://docs.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#X509TrustManager
- tmf.init(keystore);
- managers = tmf.getTrustManagers();
- }
-
- SSLContext sslContext = SSLContext.getInstance("TLS");
- sslContext.init(null, managers, null);
- SSLEngine sslEngine = sslContext.createSSLEngine();
- sslEngine.setUseClientMode(true);
-
- List<String> enabledProtocols = new ArrayList<String>();
- for (String protocol : sslEngine.getEnabledProtocols()) {
- if ((includeProtocols.isEmpty() || includeProtocols.contains(protocol))
- && !excludeProtocols.contains(protocol)) {
- enabledProtocols.add(protocol);
- }
- }
- sslEngine.setEnabledProtocols(enabledProtocols.toArray(new String[0]));
-
- List<String> enabledCipherSuites = new ArrayList<String>();
- for (String suite : sslEngine.getEnabledCipherSuites()) {
- if ((includeCipherSuites.isEmpty() || includeCipherSuites.contains(suite))
- && !excludeCipherSuites.contains(suite)) {
- enabledCipherSuites.add(suite);
- }
- }
- sslEngine.setEnabledCipherSuites(enabledCipherSuites.toArray(new String[0]));
-
- logger.info("SSLEngine protocols enabled: " + Arrays.asList(sslEngine.getEnabledProtocols()));
- logger.info("SSLEngine cipher suites enabled: " + Arrays.asList(sslEngine.getEnabledProtocols()));
- return sslEngine;
- } else {
- return null;
- }
-
- } catch (Exception ex) {
- logger.error("Cannot create SSL channel", ex);
- throw new RuntimeException("Cannot create SSL channel", ex);
- }
- }
-
- /**
- * Permissive trust manager accepting any certificate
- */
- private static class PermissiveTrustManager implements X509TrustManager {
- @Override
- public void checkClientTrusted(X509Certificate[] certs, String s) {
- // nothing
- }
-
- @Override
- public void checkServerTrusted(X509Certificate[] certs, String s) {
- // nothing
- }
-
- @Override
- public X509Certificate[] getAcceptedIssuers() {
- return new X509Certificate[0];
- }
- }
-}
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/api/RpcClient.java b/flume-ng-sdk/src/main/java/org/apache/flume/api/RpcClient.java
deleted file mode 100644
index 15b4ad7b9..000000000
--- a/flume-ng-sdk/src/main/java/org/apache/flume/api/RpcClient.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.api;
-
-import java.util.List;
-
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.FlumeException;
-
-/**
- * <p>Public client interface for sending data to Flume.</p>
- *
- * <p>This interface is intended not to change incompatibly for Flume 1.x.</p>
- *
- * <p><strong>Note:</strong> It is recommended for applications to construct
- * {@link RpcClient} instances using the {@link RpcClientFactory} class,
- * instead of using builders associated with a particular implementation class.
- * </p>
- *
- * @see org.apache.flume.api.RpcClientFactory
- */
-public interface RpcClient {
-
- /**
- * Returns the maximum number of {@link Event events} that may be batched
- * at once by {@link #appendBatch(List) appendBatch()}.
- */
- public int getBatchSize();
-
- /**
- * <p>Send a single {@link Event} to the associated Flume source.</p>
- *
- * <p>This method blocks until the RPC returns or until the request times out.
- * </p>
- *
- * <p><strong>Note:</strong> If this method throws an
- * {@link EventDeliveryException}, there is no way to recover and the
- * application must invoke {@link #close()} on this object to clean up system
- * resources.</p>
- *
- * @param event
- *
- * @throws EventDeliveryException when an error prevents event delivery.
- */
- public void append(Event event) throws EventDeliveryException;
-
- /**
- * <p>Send a list of {@linkplain Event events} to the associated Flume source.
- * </p>
- *
- * <p>This method blocks until the RPC returns or until the request times out.
- * </p>
- *
- * <p>It is strongly recommended that the number of events in the List be no
- * more than {@link #getBatchSize()}. If it is more, multiple RPC calls will
- * be required, and the likelihood of duplicate Events being stored will
- * increase.</p>
- *
- * <p><strong>Note:</strong> If this method throws an
- * {@link EventDeliveryException}, there is no way to recover and the
- * application must invoke {@link #close()} on this object to clean up system
- * resources.</p>
- *
- * @param events List of events to send
- *
- * @throws EventDeliveryException when an error prevents event delivery.
- */
- public void appendBatch(List<Event> events) throws
- EventDeliveryException;
-
- /**
- * <p>Returns {@code true} if this object appears to be in a usable state, and
- * it returns {@code false} if this object is permanently disabled.</p>
- *
- * <p>If this method returns {@code false}, an application must call
- * {@link #close()} on this object to clean up system resources.</p>
- */
- public boolean isActive();
-
- /**
- * <p>Immediately closes the client so that it may no longer be used.</p>
- *
- * <p><strong>Note:</strong> This method MUST be called by applications
- * when they are done using the RPC client in order to clean up resources.</p>
- *
- * <p>Multi-threaded applications may want to gracefully stop making
- * RPC calls before calling close(). Otherwise, they risk getting
- * {@link EventDeliveryException} thrown from their in-flight calls when the
- * underlying connection is disabled.</p>
- */
- public void close() throws FlumeException;
-
-
-}
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/api/RpcClientConfigurationConstants.java b/flume-ng-sdk/src/main/java/org/apache/flume/api/RpcClientConfigurationConstants.java
deleted file mode 100644
index 621b10ebb..000000000
--- a/flume-ng-sdk/src/main/java/org/apache/flume/api/RpcClientConfigurationConstants.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.api;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Configuration constants used by the RpcClient. These configuration keys
- * can be specified via a Properties object to the appropriate method of
- * RpcClientFactory in order to obtain a customized RPC client.
- */
-public final class RpcClientConfigurationConstants {
-
- /**
- * Hosts configuration key to specify a space delimited list of named
- * hosts. For example:
- * <pre>
- * hosts = h1 h2
- * </pre>
- */
- public static final String CONFIG_HOSTS = "hosts";
-
- /**
- * Hosts prefix to specify address of a particular named host. For example
- * <pre>
- * hosts.h1 = server1.example.com:12121
- * hosts.h2 = server2.example.com:12121
- * </pre>
- */
- public static final String CONFIG_HOSTS_PREFIX = "hosts.";
-
- /**
- * Configuration key used to specify the batch size. Default batch size is
- * {@value DEFAULT_BATCH_SIZE}.
- */
- public static final String CONFIG_BATCH_SIZE = "batch-size";
-
- /**
- * Configuration key to specify connection timeout in milliseconds. The
- * default connection timeout is {@value DEFAULT_CONNECT_TIMEOUT_MILLIS}.
- */
- public static final String CONFIG_CONNECT_TIMEOUT = "connect-timeout";
-
- /**
- * Configuration key to specify request timeout in milliseconds. The
- * default request timeout is {@value DEFAULT_REQUEST_TIMEOUT_MILLIS}.
- */
- public static final String CONFIG_REQUEST_TIMEOUT = "request-timeout";
-
- /**
- * Default batch size.
- */
- public static final Integer DEFAULT_BATCH_SIZE = 100;
-
- /**
- * Default connection, handshake, and initial request timeout in milliseconds.
- */
- public static final long DEFAULT_CONNECT_TIMEOUT_MILLIS =
- TimeUnit.MILLISECONDS.convert(20, TimeUnit.SECONDS);
-
- /**
- * Default request timeout in milliseconds.
- */
- public static final long DEFAULT_REQUEST_TIMEOUT_MILLIS =
- TimeUnit.MILLISECONDS.convert(20, TimeUnit.SECONDS);
-
- /**
- * Maximum attempts to be made by the FailoverRpcClient in case of
- * failures.
- */
- public static final String CONFIG_MAX_ATTEMPTS = "max-attempts";
-
- /**
- * Configuration key to specify the RpcClient type to be used. The available
- * values are <tt>DEFAULT</tt> which results in the creation of a regular
- * <tt>NettyAvroRpcClient</tt> and <tt>DEFAULT_FAILOVER</tt> which results
- * in the creation of a failover client implementation on top of multiple
- * <tt>NettyAvroRpcClient</tt>s. The default value of this configuration
- * is {@value #DEFAULT_CLIENT_TYPE}.
- *
- */
- public static final String CONFIG_CLIENT_TYPE = "client.type";
-
- /**
- * The default client type to be created if no explicit type is specified.
- */
- public static final String DEFAULT_CLIENT_TYPE =
- RpcClientFactory.ClientType.DEFAULT.name();
-
- /**
- * The selector type used by the <tt>LoadBalancingRpcClient</tt>. This
- * value of this setting could be either <tt>round_robin</tt>,
- * <tt>random</tt>, or the fully qualified name class that implements the
- * <tt>LoadBalancingRpcClient.HostSelector</tt> interface.
- */
- public static final String CONFIG_HOST_SELECTOR =
- "host-selector";
-
- public static final String HOST_SELECTOR_ROUND_ROBIN = "ROUND_ROBIN";
- public static final String HOST_SELECTOR_RANDOM = "RANDOM";
-
- public static final String CONFIG_MAX_BACKOFF = "maxBackoff";
- public static final String CONFIG_BACKOFF = "backoff";
- public static final String DEFAULT_BACKOFF = "false";
-
- /**
- * Maximum number of connections each Thrift Rpc client can open to a given
- * host.
- */
- public static final String CONFIG_CONNECTION_POOL_SIZE = "maxConnections";
- public static final int DEFAULT_CONNECTION_POOL_SIZE = 5;
-
- /**
- * The following are const for the NettyAvro Client. To enable compression
- * and set a compression level
- */
- public static final String CONFIG_COMPRESSION_TYPE = "compression-type";
- public static final String CONFIG_COMPRESSION_LEVEL = "compression-level";
- public static final int DEFAULT_COMPRESSION_LEVEL = 6;
-
- /**
- * Configuration constants for SSL support
- */
- public static final String CONFIG_SSL = "ssl";
- public static final String CONFIG_TRUST_ALL_CERTS = "trust-all-certs";
- public static final String CONFIG_TRUSTSTORE = "truststore";
- public static final String CONFIG_TRUSTSTORE_PASSWORD = "truststore-password";
- public static final String CONFIG_TRUSTSTORE_TYPE = "truststore-type";
- public static final String CONFIG_EXCLUDE_PROTOCOLS = "exclude-protocols";
- public static final String CONFIG_INCLUDE_PROTOCOLS = "include-protocols";
- public static final String CONFIG_EXCLUDE_CIPHER_SUITES = "exclude-cipher-suites";
- public static final String CONFIG_INCLUDE_CIPHER_SUITES = "include-cipher-suites";
-
- public static final String KERBEROS_KEY = "kerberos";
-
- /**
- * Configuration constants for the NettyAvroRpcClient
- * NioClientSocketChannelFactory
- */
- public static final String MAX_IO_WORKERS = "maxIoWorkers";
-
- private RpcClientConfigurationConstants() {
- // disable explicit object creation
- }
-}
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/api/RpcClientFactory.java b/flume-ng-sdk/src/main/java/org/apache/flume/api/RpcClientFactory.java
deleted file mode 100644
index 4326b579a..000000000
--- a/flume-ng-sdk/src/main/java/org/apache/flume/api/RpcClientFactory.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.api;
-
-import org.apache.flume.FlumeException;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.Reader;
-import java.util.Locale;
-import java.util.Properties;
-
-/**
- * Factory class to construct Flume {@link RPCClient} implementations.
- */
-public class RpcClientFactory {
-
- /**
- * Returns an instance of {@link RpcClient}, optionally with failover.
- * To create a failover client, the properties object should have a
- * property <tt>client.type</tt> which has the value "failover". The client
- * connects to hosts specified by <tt>hosts</tt> property in given properties.
- *
- * @see org.apache.flume.api.FailoverRpcClient
- * <p>
- * If no <tt>client.type</tt> is specified, a default client that connects to
- * single host at a given port is created.(<tt>type</tt> can also simply be
- * <tt>DEFAULT</tt> for the default client).
- *
- * @see org.apache.flume.api.NettyAvroClient
- *
- * @param properties The properties to instantiate the client with.
- * @throws FlumeException
- */
- @SuppressWarnings("unchecked")
- public static RpcClient getInstance(Properties properties)
- throws FlumeException {
- String type = null;
- type = properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_CLIENT_TYPE);
- if (type == null || type.isEmpty()) {
- type = ClientType.DEFAULT.getClientClassName();
- }
- Class<? extends AbstractRpcClient> clazz;
- AbstractRpcClient client;
- try {
- String clientClassType = type;
- ClientType clientType = null;
- try {
- clientType = ClientType.valueOf(type.toUpperCase(Locale.ENGLISH));
- } catch (IllegalArgumentException e) {
- clientType = ClientType.OTHER;
- }
- if (!clientType.equals(ClientType.OTHER)) {
- clientClassType = clientType.getClientClassName();
- }
- clazz =
- (Class<? extends AbstractRpcClient>) Class.forName(clientClassType);
- } catch (ClassNotFoundException e) {
- throw new FlumeException("No such client!", e);
- }
-
- try {
- client = clazz.newInstance();
- } catch (InstantiationException e) {
- throw new FlumeException("Cannot instantiate client. " +
- "Exception follows:", e);
- } catch (IllegalAccessException e) {
- throw new FlumeException("Cannot instantiate client. " +
- "Exception follows:", e);
- }
- client.configure(properties);
- return client;
-
- }
-
- /**
- * Delegates to {@link #getInstance(Properties props)}, given a File path
- * to a {@link Properties} file.
- * @param propertiesFile Valid properties file
- * @return RpcClient configured according to the given Properties file.
- * @throws FileNotFoundException If the file cannot be found
- * @throws IOException If there is an IO error
- */
- public static RpcClient getInstance(File propertiesFile)
- throws FileNotFoundException, IOException {
- Reader reader = new FileReader(propertiesFile);
- Properties props = new Properties();
- props.load(reader);
- return getInstance(props);
- }
-
- /**
- * Deprecated. Use
- * {@link #getDefaultInstance(String, Integer)} instead.
- * @throws FlumeException
- * @deprecated
- */
- @Deprecated
- public static RpcClient getInstance(String hostname, Integer port)
- throws FlumeException {
- return getDefaultInstance(hostname, port);
- }
-
- /**
- * Returns an instance of {@link RpcClient} connected to the specified
- * {@code hostname} and {@code port}.
- * @throws FlumeException
- */
- public static RpcClient getDefaultInstance(String hostname, Integer port)
- throws FlumeException {
- return getDefaultInstance(hostname, port, 0);
-
- }
-
- /**
- * Deprecated. Use
- * {@link #getDefaultInstance(String, Integer, Integer)}
- * instead.
- * @throws FlumeException
- * @deprecated
- */
- @Deprecated
- public static RpcClient getInstance(String hostname, Integer port,
- Integer batchSize) throws FlumeException {
- return getDefaultInstance(hostname, port, batchSize);
- }
-
- /**
- * Returns an instance of {@link RpcClient} connected to the specified
- * {@code hostname} and {@code port} with the specified {@code batchSize}.
- * @throws FlumeException
- */
- public static RpcClient getDefaultInstance(String hostname, Integer port,
- Integer batchSize) throws FlumeException {
-
- if (hostname == null) {
- throw new NullPointerException("hostname must not be null");
- }
- if (port == null) {
- throw new NullPointerException("port must not be null");
- }
- if (batchSize == null) {
- throw new NullPointerException("batchSize must not be null");
- }
-
- Properties props = new Properties();
- props.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS, "h1");
- props.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + "h1",
- hostname + ":" + port.intValue());
- props.setProperty(RpcClientConfigurationConstants.CONFIG_BATCH_SIZE, batchSize.toString());
- NettyAvroRpcClient client = new NettyAvroRpcClient();
- client.configure(props);
- return client;
- }
-
- /**
- * Return an {@linkplain RpcClient} that uses Thrift for communicating with
- * the next hop. The next hop must have a ThriftSource listening on the
- * specified port.
- * @param hostname - The hostname of the next hop.
- * @param port - The port on which the ThriftSource is listening
- * @param batchSize - batch size of each transaction.
- * @return an {@linkplain RpcClient} which uses thrift configured with the
- * given parameters.
- */
- public static RpcClient getThriftInstance(String hostname, Integer port, Integer batchSize) {
- if (hostname == null) {
- throw new NullPointerException("hostname must not be null");
- }
- if (port == null) {
- throw new NullPointerException("port must not be null");
- }
- if (batchSize == null) {
- throw new NullPointerException("batchSize must not be null");
- }
-
- Properties props = new Properties();
- props.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS, "h1");
- props.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + "h1",
- hostname + ":" + port.intValue());
- props.setProperty(RpcClientConfigurationConstants.CONFIG_BATCH_SIZE, batchSize.toString());
- ThriftRpcClient client = new ThriftRpcClient();
- client.configure(props);
- return client;
- }
-
- /**
- * Return an {@linkplain RpcClient} that uses Thrift for communicating with
- * the next hop. The next hop must have a ThriftSource listening on the
- * specified port. This will use the default batch size. See {@linkplain
- * RpcClientConfigurationConstants}
- * @param hostname - The hostname of the next hop.
- * @param port - The port on which the ThriftSource is listening
- * @return - An {@linkplain RpcClient} which uses thrift configured with the
- * given parameters.
- */
- public static RpcClient getThriftInstance(String hostname, Integer port) {
- return getThriftInstance(hostname, port, RpcClientConfigurationConstants
- .DEFAULT_BATCH_SIZE);
- }
-
- /**
- * Return an {@linkplain RpcClient} that uses Thrift for communicating with
- * the next hop.
- * @param props
- * @return - An {@linkplain RpcClient} which uses thrift configured with the
- * given parameters.
- */
- public static RpcClient getThriftInstance(Properties props) {
- props.setProperty(RpcClientConfigurationConstants.CONFIG_CLIENT_TYPE,
- ClientType.THRIFT.clientClassName);
- return getInstance(props);
- }
-
- public static enum ClientType {
- OTHER(null),
- DEFAULT(NettyAvroRpcClient.class.getCanonicalName()),
- DEFAULT_FAILOVER(FailoverRpcClient.class.getCanonicalName()),
- DEFAULT_LOADBALANCE(LoadBalancingRpcClient.class.getCanonicalName()),
- THRIFT(ThriftRpcClient.class.getCanonicalName());
-
-
- private final String clientClassName;
-
- private ClientType(String className) {
- this.clientClassName = className;
- }
-
- protected String getClientClassName() {
- return this.clientClassName;
- }
-
- }
-}
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/api/SSLContextAwareAbstractRpcClient.java b/flume-ng-sdk/src/main/java/org/apache/flume/api/SSLContextAwareAbstractRpcClient.java
deleted file mode 100644
index 34557f6ab..000000000
--- a/flume-ng-sdk/src/main/java/org/apache/flume/api/SSLContextAwareAbstractRpcClient.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.api;
-
-import java.util.Arrays;
-import java.util.LinkedHashSet;
-import java.util.Objects;
-import java.util.Properties;
-import java.util.Set;
-
-import org.apache.flume.FlumeException;
-import org.apache.flume.util.SSLUtil;
-
-public abstract class SSLContextAwareAbstractRpcClient extends AbstractRpcClient {
- protected boolean enableSsl;
- protected boolean trustAllCerts;
- protected String truststore;
- protected String truststorePassword;
- protected String truststoreType;
- protected final Set<String> excludeProtocols = new LinkedHashSet<>(Arrays.asList("SSLv3"));
- protected final Set<String> includeProtocols = new LinkedHashSet<>();
- protected final Set<String> excludeCipherSuites = new LinkedHashSet<>();
- protected final Set<String> includeCipherSuites = new LinkedHashSet<>();
-
- protected void configureSSL(Properties properties) throws FlumeException {
- enableSsl = Boolean.parseBoolean(properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_SSL));
- trustAllCerts = Boolean.parseBoolean(properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_TRUST_ALL_CERTS));
- truststore = properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_TRUSTSTORE, SSLUtil.getGlobalTruststorePath());
- truststorePassword = properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_TRUSTSTORE_PASSWORD,
- SSLUtil.getGlobalTruststorePassword());
- truststoreType = properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_TRUSTSTORE_TYPE,
- SSLUtil.getGlobalTruststoreType("JKS"));
- parseList(properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_EXCLUDE_PROTOCOLS,
- SSLUtil.getGlobalExcludeProtocols()),
- excludeProtocols);
- parseList(properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_INCLUDE_PROTOCOLS,
- SSLUtil.getGlobalIncludeProtocols()),
- includeProtocols);
- parseList(properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_EXCLUDE_CIPHER_SUITES,
- SSLUtil.getGlobalExcludeCipherSuites()),
- excludeCipherSuites);
- parseList(properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_INCLUDE_CIPHER_SUITES,
- SSLUtil.getGlobalIncludeCipherSuites()),
- includeCipherSuites);
- }
-
- private void parseList(String value, Set<String> set) {
- if (Objects.nonNull(value)) {
- set.addAll(Arrays.asList(value.split(" ")));
- }
- }
-}
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/api/ThriftRpcClient.java b/flume-ng-sdk/src/main/java/org/apache/flume/api/ThriftRpcClient.java
deleted file mode 100644
index 56053bc8b..000000000
--- a/flume-ng-sdk/src/main/java/org/apache/flume/api/ThriftRpcClient.java
+++ /dev/null
@@ -1,542 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.api;
-
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.FlumeException;
-import org.apache.flume.thrift.Status;
-import org.apache.flume.thrift.ThriftFlumeEvent;
-import org.apache.flume.thrift.ThriftSourceProtocol;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.protocol.TCompactProtocol;
-import org.apache.thrift.transport.TSocket;
-import org.apache.thrift.transport.TTransport;
-import org.apache.thrift.transport.layered.TFastFramedTransport;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSocket;
-import javax.net.ssl.SSLSocketFactory;
-import javax.net.ssl.TrustManagerFactory;
-import java.io.FileInputStream;
-import java.nio.ByteBuffer;
-import java.security.KeyStore;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Properties;
-import java.util.Queue;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-public class ThriftRpcClient extends SSLContextAwareAbstractRpcClient {
- private static final Logger LOGGER = LoggerFactory.getLogger(ThriftRpcClient.class);
-
- /**
- * Config param for the thrift protocol to use.
- */
- public static final String CONFIG_PROTOCOL = "protocol";
- public static final String BINARY_PROTOCOL = "binary";
- public static final String COMPACT_PROTOCOL = "compact";
-
- private long requestTimeout;
- private final Lock stateLock;
- private State connState;
- private String hostname;
- private int port;
- private ConnectionPoolManager connectionManager;
- private final ExecutorService callTimeoutPool;
- private final AtomicLong threadCounter;
- private final Random random = new Random();
- private String protocol;
-
- public ThriftRpcClient() {
- stateLock = new ReentrantLock(true);
- connState = State.INIT;
-
- threadCounter = new AtomicLong(0);
- // OK to use cached threadpool, because this is simply meant to timeout
- // the calls - and is IO bound.
- callTimeoutPool = Executors.newCachedThreadPool(new ThreadFactory() {
- @Override
- public Thread newThread(Runnable r) {
- Thread t = new Thread(r);
- t.setName("Flume Thrift RPC thread - " + String.valueOf(threadCounter.incrementAndGet()));
- return t;
- }
- });
- }
-
- @Override
- public void append(Event event) throws EventDeliveryException {
- // Thrift IPC client is not thread safe, so don't allow state changes or
- // client.append* calls unless the lock is acquired.
- ClientWrapper client = null;
- boolean destroyedClient = false;
- try {
- if (!isActive()) {
- throw new EventDeliveryException("Client was closed due to error. " +
- "Please create a new client");
- }
- client = connectionManager.checkout();
- final ThriftFlumeEvent thriftEvent = new ThriftFlumeEvent(event
- .getHeaders(), ByteBuffer.wrap(event.getBody()));
- doAppend(client, thriftEvent).get(requestTimeout, TimeUnit.MILLISECONDS);
- } catch (Throwable e) {
- if (e instanceof ExecutionException) {
- Throwable cause = e.getCause();
- if (cause instanceof EventDeliveryException) {
- throw (EventDeliveryException) cause;
- } else if (cause instanceof TimeoutException) {
- throw new EventDeliveryException("Append call timeout", cause);
- }
- }
- destroyedClient = true;
- // If destroy throws, we still don't want to reuse the client, so mark it
- // as destroyed before we actually do.
- if (client != null) {
- connectionManager.destroy(client);
- }
- if (e instanceof Error) {
- throw (Error) e;
- } else if (e instanceof RuntimeException) {
- throw (RuntimeException) e;
- }
- throw new EventDeliveryException("Failed to send event. ", e);
- } finally {
- if (client != null && !destroyedClient) {
- connectionManager.checkIn(client);
- }
- }
- }
-
- @Override
- public void appendBatch(List<Event> events) throws EventDeliveryException {
- // Thrift IPC client is not thread safe, so don't allow state changes or
- // client.append* calls unless the lock is acquired.
- ClientWrapper client = null;
- boolean destroyedClient = false;
- try {
- if (!isActive()) {
- throw new EventDeliveryException("Client was closed " +
- "due to error or is not yet configured.");
- }
- client = connectionManager.checkout();
- final List<ThriftFlumeEvent> thriftFlumeEvents = new ArrayList
- <ThriftFlumeEvent>();
- Iterator<Event> eventsIter = events.iterator();
- while (eventsIter.hasNext()) {
- thriftFlumeEvents.clear();
- for (int i = 0; i < batchSize && eventsIter.hasNext(); i++) {
- Event event = eventsIter.next();
- thriftFlumeEvents.add(new ThriftFlumeEvent(event.getHeaders(),
- ByteBuffer.wrap(event.getBody())));
- }
- if (!thriftFlumeEvents.isEmpty()) {
- doAppendBatch(client, thriftFlumeEvents).get(requestTimeout,
- TimeUnit.MILLISECONDS);
- }
- }
- } catch (Throwable e) {
- if (e instanceof ExecutionException) {
- Throwable cause = e.getCause();
- if (cause instanceof EventDeliveryException) {
- throw (EventDeliveryException) cause;
- } else if (cause instanceof TimeoutException) {
- throw new EventDeliveryException("Append call timeout", cause);
- }
- }
- destroyedClient = true;
- // If destroy throws, we still don't want to reuse the client, so mark it
- // as destroyed before we actually do.
- if (client != null) {
- connectionManager.destroy(client);
- }
- if (e instanceof Error) {
- throw (Error) e;
- } else if (e instanceof RuntimeException) {
- throw (RuntimeException) e;
- }
- throw new EventDeliveryException("Failed to send event. ", e);
- } finally {
- if (client != null && !destroyedClient) {
- connectionManager.checkIn(client);
- }
- }
- }
-
- private Future<Void> doAppend(final ClientWrapper client,
- final ThriftFlumeEvent e) throws Exception {
-
- return callTimeoutPool.submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- Status status = client.client.append(e);
- if (status != Status.OK) {
- throw new EventDeliveryException("Failed to deliver events. Server " +
- "returned status : " + status.name());
- }
- return null;
- }
- });
- }
-
- private Future<Void> doAppendBatch(final ClientWrapper client,
- final List<ThriftFlumeEvent> e) throws Exception {
-
- return callTimeoutPool.submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- Status status = client.client.appendBatch(e);
- if (status != Status.OK) {
- throw new EventDeliveryException("Failed to deliver events. Server " +
- "returned status : " + status.name());
- }
- return null;
- }
- });
- }
-
- @Override
- public boolean isActive() {
- stateLock.lock();
- try {
- return (connState == State.READY);
- } finally {
- stateLock.unlock();
- }
- }
-
- @Override
- public void close() throws FlumeException {
- try {
- //Do not release this, because this client is not to be used again
- stateLock.lock();
- connState = State.DEAD;
- connectionManager.closeAll();
- callTimeoutPool.shutdown();
- if (!callTimeoutPool.awaitTermination(5, TimeUnit.SECONDS)) {
- callTimeoutPool.shutdownNow();
- }
- } catch (Throwable ex) {
- if (ex instanceof Error) {
- throw (Error) ex;
- } else if (ex instanceof RuntimeException) {
- throw (RuntimeException) ex;
- }
- throw new FlumeException("Failed to close RPC client. ", ex);
- } finally {
- stateLock.unlock();
- }
- }
-
- @Override
- protected void configure(Properties properties) throws FlumeException {
- if (isActive()) {
- throw new FlumeException("Attempting to re-configured an already " +
- "configured client!");
- }
- stateLock.lock();
- try {
- HostInfo host = HostInfo.getHostInfoList(properties).get(0);
- hostname = host.getHostName();
- port = host.getPortNumber();
- protocol = properties.getProperty(CONFIG_PROTOCOL);
- if (protocol == null) {
- // default is to use the compact protocol.
- protocol = COMPACT_PROTOCOL;
- }
- // check in case that garbage was put in.
- if (!(protocol.equalsIgnoreCase(BINARY_PROTOCOL) ||
- protocol.equalsIgnoreCase(COMPACT_PROTOCOL))) {
- LOGGER.warn("'binary' or 'compact' are the only valid Thrift protocol types to "
- + "choose from. Defaulting to 'compact'.");
- protocol = COMPACT_PROTOCOL;
- }
- batchSize = parseBatchSize(properties);
- requestTimeout = Long.parseLong(properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_REQUEST_TIMEOUT,
- String.valueOf(
- RpcClientConfigurationConstants.DEFAULT_REQUEST_TIMEOUT_MILLIS)));
- if (requestTimeout < 1000) {
- LOGGER.warn("Request timeout specified less than 1s. " +
- "Using default value instead.");
- requestTimeout =
- RpcClientConfigurationConstants.DEFAULT_REQUEST_TIMEOUT_MILLIS;
- }
- int connectionPoolSize = Integer.parseInt(properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_CONNECTION_POOL_SIZE,
- String.valueOf(RpcClientConfigurationConstants
- .DEFAULT_CONNECTION_POOL_SIZE)));
- if (connectionPoolSize < 1) {
- LOGGER.warn("Connection Pool Size specified is less than 1. " +
- "Using default value instead.");
- connectionPoolSize = RpcClientConfigurationConstants
- .DEFAULT_CONNECTION_POOL_SIZE;
- }
- configureSSL(properties);
-
- connectionManager = new ConnectionPoolManager(connectionPoolSize);
- connState = State.READY;
- } catch (Throwable ex) {
- //Failed to configure, kill the client.
- connState = State.DEAD;
- if (ex instanceof Error) {
- throw (Error) ex;
- } else if (ex instanceof RuntimeException) {
- throw (RuntimeException) ex;
- }
- throw new FlumeException("Error while configuring RpcClient. ", ex);
- } finally {
- stateLock.unlock();
- }
- }
-
- private static enum State {
- INIT, READY, DEAD
- }
-
- protected TTransport getTransport(TSocket tsocket) throws Exception {
- return new TFastFramedTransport(tsocket);
- }
-
- /**
- * Wrapper around a client and transport, so we can clean up when this
- * client gets closed.
- */
- private class ClientWrapper {
- public final ThriftSourceProtocol.Client client;
- public final TTransport transport;
- private final int hashCode;
-
- public ClientWrapper() throws Exception {
- TSocket tsocket;
- if (enableSsl) {
- // JDK6's factory doesn't appear to pass the protocol onto the Socket
- // properly so we have to do some magic to make sure that happens.
- // Not an issue in JDK7 Lifted from thrift-0.9.1 to make the SSLContext
- SSLContext sslContext = createSSLContext(truststore, truststorePassword,
- truststoreType);
-
- // Create the factory from it
- SSLSocketFactory sslSockFactory = sslContext.getSocketFactory();
-
- // Create the TSocket from that
- tsocket = createSSLSocket(
- sslSockFactory, hostname, port, 120000, excludeProtocols,
- includeProtocols, excludeCipherSuites, includeCipherSuites);
- } else {
- tsocket = new TSocket(hostname, port);
- }
-
- transport = getTransport(tsocket);
-
- // The transport is already open for SSL as part of TSSLTransportFactory.getClientSocket
- if (!transport.isOpen()) {
- transport.open();
- }
- if (protocol.equals(BINARY_PROTOCOL)) {
- LOGGER.info("Using TBinaryProtocol");
- client = new ThriftSourceProtocol.Client(new TBinaryProtocol(transport));
- } else {
- LOGGER.info("Using TCompactProtocol");
- client = new ThriftSourceProtocol.Client(new TCompactProtocol(transport));
- }
- // Not a great hash code, but since this class is immutable and there
- // is at most one instance of the components of this class,
- // this works fine [If the objects are equal, hash code is the same]
- hashCode = random.nextInt();
- }
-
- public boolean equals(Object o) {
- if (o == null) {
- return false;
- }
- // Since there is only one wrapper with any given client,
- // direct comparison is good enough.
- if (this == o) {
- return true;
- }
- return false;
- }
-
- public int hashCode() {
- return hashCode;
- }
- }
-
- private class ConnectionPoolManager {
- private final Queue<ClientWrapper> availableClients;
- private final Set<ClientWrapper> checkedOutClients;
- private final int maxPoolSize;
- private int currentPoolSize;
- private final Lock poolLock;
- private final Condition availableClientsCondition;
-
- public ConnectionPoolManager(int poolSize) {
- this.maxPoolSize = poolSize;
- availableClients = new LinkedList<ClientWrapper>();
- checkedOutClients = new HashSet<ClientWrapper>();
- poolLock = new ReentrantLock();
- availableClientsCondition = poolLock.newCondition();
- currentPoolSize = 0;
- }
-
- public ClientWrapper checkout() throws Exception {
-
- ClientWrapper ret = null;
- poolLock.lock();
- try {
- if (availableClients.isEmpty() && currentPoolSize < maxPoolSize) {
- ret = new ClientWrapper();
- currentPoolSize++;
- checkedOutClients.add(ret);
- return ret;
- }
- while (availableClients.isEmpty()) {
- availableClientsCondition.await();
- }
- ret = availableClients.poll();
- checkedOutClients.add(ret);
- } finally {
- poolLock.unlock();
- }
- return ret;
- }
-
- public void checkIn(ClientWrapper client) {
- poolLock.lock();
- try {
- availableClients.add(client);
- checkedOutClients.remove(client);
- availableClientsCondition.signal();
- } finally {
- poolLock.unlock();
- }
- }
-
- public void destroy(ClientWrapper client) {
- poolLock.lock();
- try {
- checkedOutClients.remove(client);
- currentPoolSize--;
- } finally {
- poolLock.unlock();
- }
- client.transport.close();
- }
-
- public void closeAll() {
- poolLock.lock();
- try {
- for (ClientWrapper c : availableClients) {
- c.transport.close();
- currentPoolSize--;
- }
- // Be cruel and close even the checked out clients. The threads writing
- // using these will now get an exception.
- for (ClientWrapper c : checkedOutClients) {
- c.transport.close();
- currentPoolSize--;
- }
- } finally {
- poolLock.unlock();
- }
- }
- }
-
- /**
- * Lifted from ACCUMULO-3318 - Lifted from TSSLTransportFactory in Thrift-0.9.1.
- * The method to create a client socket with an SSLContextFactory object is not visible to us.
- * Have to use * SslConnectionParams instead of TSSLTransportParameters because no getters exist
- * on TSSLTransportParameters.
- */
- private static SSLContext createSSLContext(String truststore,
- String truststorePassword,
- String truststoreType) throws FlumeException {
- SSLContext ctx;
- try {
- ctx = SSLContext.getInstance("TLS");
- TrustManagerFactory tmf;
- tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
- KeyStore ts = null;
- if (truststore != null && truststoreType != null) {
- ts = KeyStore.getInstance(truststoreType);
- ts.load(new FileInputStream(truststore),
- truststorePassword != null ? truststorePassword.toCharArray() : null);
- tmf.init(ts);
- }
-
- tmf.init(ts);
- ctx.init(null, tmf.getTrustManagers(), null);
-
- } catch (Exception e) {
- throw new FlumeException("Error creating the transport", e);
- }
- return ctx;
- }
-
- private static TSocket createSSLSocket(SSLSocketFactory factory, String host,
- int port, int timeout, Set<String> excludeProtocols, Set<String> includeProtocols,
- Set<String> excludeCipherSuites, Set<String> includeCipherSuites)
- throws FlumeException {
- try {
- SSLSocket socket = (SSLSocket) factory.createSocket(host, port);
- socket.setSoTimeout(timeout);
-
- List<String> enabledProtocols = new ArrayList<String>();
- for (String protocol : socket.getEnabledProtocols()) {
- if ((includeProtocols.isEmpty() || includeProtocols.contains(protocol))
- && !excludeProtocols.contains(protocol)) {
- enabledProtocols.add(protocol);
- }
- }
- socket.setEnabledProtocols(enabledProtocols.toArray(new String[0]));
-
- List<String> enabledCipherSuites = new ArrayList<String>();
- for (String suite : socket.getEnabledCipherSuites()) {
- if ((includeCipherSuites.isEmpty() || includeCipherSuites.contains(suite))
- && !excludeCipherSuites.contains(suite)) {
- enabledCipherSuites.add(suite);
- }
- }
- socket.setEnabledCipherSuites(enabledCipherSuites.toArray(new String[0]));
-
- return new TSocket(socket);
- } catch (Exception e) {
- throw new FlumeException("Could not connect to " + host + " on port " + port, e);
- }
- }
-}
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/thrift/Status.java b/flume-ng-sdk/src/main/java/org/apache/flume/thrift/Status.java
deleted file mode 100644
index 9118d35ed..000000000
--- a/flume-ng-sdk/src/main/java/org/apache/flume/thrift/Status.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.flume.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum Status implements org.apache.thrift.TEnum {
- OK(0),
- FAILED(1),
- ERROR(2),
- UNKNOWN(3);
-
- private final int value;
-
- private Status(int value) {
- this.value = value;
- }
-
- /**
- * Get the integer value of this enum value, as defined in the Thrift IDL.
- */
- public int getValue() {
- return value;
- }
-
- /**
- * Find a the enum type by its integer value, as defined in the Thrift IDL.
- * @return null if the value is not found.
- */
- public static Status findByValue(int value) {
- switch (value) {
- case 0:
- return OK;
- case 1:
- return FAILED;
- case 2:
- return ERROR;
- case 3:
- return UNKNOWN;
- default:
- return null;
- }
- }
-}
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/thrift/ThriftFlumeEvent.java b/flume-ng-sdk/src/main/java/org/apache/flume/thrift/ThriftFlumeEvent.java
deleted file mode 100644
index 1eeea84d8..000000000
--- a/flume-ng-sdk/src/main/java/org/apache/flume/thrift/ThriftFlumeEvent.java
+++ /dev/null
@@ -1,572 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.flume.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2017-09-01")
-public class ThriftFlumeEvent implements org.apache.thrift.TBase<ThriftFlumeEvent, ThriftFlumeEvent._Fields>, java.io.Serializable, Cloneable, Comparable<ThriftFlumeEvent> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ThriftFlumeEvent");
-
- private static final org.apache.thrift.protocol.TField HEADERS_FIELD_DESC = new org.apache.thrift.protocol.TField("headers", org.apache.thrift.protocol.TType.MAP, (short)1);
- private static final org.apache.thrift.protocol.TField BODY_FIELD_DESC = new org.apache.thrift.protocol.TField("body", org.apache.thrift.protocol.TType.STRING, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new ThriftFlumeEventStandardSchemeFactory());
- schemes.put(TupleScheme.class, new ThriftFlumeEventTupleSchemeFactory());
- }
-
- public Map<String,String> headers; // required
- public ByteBuffer body; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- HEADERS((short)1, "headers"),
- BODY((short)2, "body");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // HEADERS
- return HEADERS;
- case 2: // BODY
- return BODY;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.HEADERS, new org.apache.thrift.meta_data.FieldMetaData("headers", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- tmpMap.put(_Fields.BODY, new org.apache.thrift.meta_data.FieldMetaData("body", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ThriftFlumeEvent.class, metaDataMap);
- }
-
- public ThriftFlumeEvent() {
- }
-
- public ThriftFlumeEvent(
- Map<String,String> headers,
- ByteBuffer body)
- {
- this();
- this.headers = headers;
- this.body = org.apache.thrift.TBaseHelper.copyBinary(body);
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public ThriftFlumeEvent(ThriftFlumeEvent other) {
- if (other.isSetHeaders()) {
- Map<String,String> __this__headers = new HashMap<String,String>(other.headers);
- this.headers = __this__headers;
- }
- if (other.isSetBody()) {
- this.body = org.apache.thrift.TBaseHelper.copyBinary(other.body);
- }
- }
-
- public ThriftFlumeEvent deepCopy() {
- return new ThriftFlumeEvent(this);
- }
-
- @Override
- public void clear() {
- this.headers = null;
- this.body = null;
- }
-
- public int getHeadersSize() {
- return (this.headers == null) ? 0 : this.headers.size();
- }
-
- public void putToHeaders(String key, String val) {
- if (this.headers == null) {
- this.headers = new HashMap<String,String>();
- }
- this.headers.put(key, val);
- }
-
- public Map<String,String> getHeaders() {
- return this.headers;
- }
-
- public ThriftFlumeEvent setHeaders(Map<String,String> headers) {
- this.headers = headers;
- return this;
- }
-
- public void unsetHeaders() {
- this.headers = null;
- }
-
- /** Returns true if field headers is set (has been assigned a value) and false otherwise */
- public boolean isSetHeaders() {
- return this.headers != null;
- }
-
- public void setHeadersIsSet(boolean value) {
- if (!value) {
- this.headers = null;
- }
- }
-
- public byte[] getBody() {
- setBody(org.apache.thrift.TBaseHelper.rightSize(body));
- return body == null ? null : body.array();
- }
-
- public ByteBuffer bufferForBody() {
- return org.apache.thrift.TBaseHelper.copyBinary(body);
- }
-
- public ThriftFlumeEvent setBody(byte[] body) {
- this.body = body == null ? (ByteBuffer)null : ByteBuffer.wrap(Arrays.copyOf(body, body.length));
- return this;
- }
-
- public ThriftFlumeEvent setBody(ByteBuffer body) {
- this.body = org.apache.thrift.TBaseHelper.copyBinary(body);
- return this;
- }
-
- public void unsetBody() {
- this.body = null;
- }
-
- /** Returns true if field body is set (has been assigned a value) and false otherwise */
- public boolean isSetBody() {
- return this.body != null;
- }
-
- public void setBodyIsSet(boolean value) {
- if (!value) {
- this.body = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case HEADERS:
- if (value == null) {
- unsetHeaders();
- } else {
- setHeaders((Map<String,String>)value);
- }
- break;
-
- case BODY:
- if (value == null) {
- unsetBody();
- } else {
- setBody((ByteBuffer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case HEADERS:
- return getHeaders();
-
- case BODY:
- return getBody();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case HEADERS:
- return isSetHeaders();
- case BODY:
- return isSetBody();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof ThriftFlumeEvent)
- return this.equals((ThriftFlumeEvent)that);
- return false;
- }
-
- public boolean equals(ThriftFlumeEvent that) {
- if (that == null)
- return false;
-
- boolean this_present_headers = true && this.isSetHeaders();
- boolean that_present_headers = true && that.isSetHeaders();
- if (this_present_headers || that_present_headers) {
- if (!(this_present_headers && that_present_headers))
- return false;
- if (!this.headers.equals(that.headers))
- return false;
- }
-
- boolean this_present_body = true && this.isSetBody();
- boolean that_present_body = true && that.isSetBody();
- if (this_present_body || that_present_body) {
- if (!(this_present_body && that_present_body))
- return false;
- if (!this.body.equals(that.body))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- List<Object> list = new ArrayList<Object>();
-
- boolean present_headers = true && (isSetHeaders());
- list.add(present_headers);
- if (present_headers)
- list.add(headers);
-
- boolean present_body = true && (isSetBody());
- list.add(present_body);
- if (present_body)
- list.add(body);
-
- return list.hashCode();
- }
-
- @Override
- public int compareTo(ThriftFlumeEvent other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetHeaders()).compareTo(other.isSetHeaders());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetHeaders()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.headers, other.headers);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetBody()).compareTo(other.isSetBody());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetBody()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.body, other.body);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("ThriftFlumeEvent(");
- boolean first = true;
-
- sb.append("headers:");
- if (this.headers == null) {
- sb.append("null");
- } else {
- sb.append(this.headers);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("body:");
- if (this.body == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.body, sb);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- if (headers == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'headers' was not present! Struct: " + toString());
- }
- if (body == null) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'body' was not present! Struct: " + toString());
- }
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class ThriftFlumeEventStandardSchemeFactory implements SchemeFactory {
- public ThriftFlumeEventStandardScheme getScheme() {
- return new ThriftFlumeEventStandardScheme();
- }
- }
-
- private static class ThriftFlumeEventStandardScheme extends StandardScheme<ThriftFlumeEvent> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, ThriftFlumeEvent struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // HEADERS
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map0 = iprot.readMapBegin();
- struct.headers = new HashMap<String,String>(2*_map0.size);
- String _key1;
- String _val2;
- for (int _i3 = 0; _i3 < _map0.size; ++_i3)
- {
- _key1 = iprot.readString();
- _val2 = iprot.readString();
- struct.headers.put(_key1, _val2);
- }
- iprot.readMapEnd();
- }
- struct.setHeadersIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // BODY
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.body = iprot.readBinary();
- struct.setBodyIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, ThriftFlumeEvent struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.headers != null) {
- oprot.writeFieldBegin(HEADERS_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.headers.size()));
- for (Map.Entry<String, String> _iter4 : struct.headers.entrySet())
- {
- oprot.writeString(_iter4.getKey());
- oprot.writeString(_iter4.getValue());
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.body != null) {
- oprot.writeFieldBegin(BODY_FIELD_DESC);
- oprot.writeBinary(struct.body);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class ThriftFlumeEventTupleSchemeFactory implements SchemeFactory {
- public ThriftFlumeEventTupleScheme getScheme() {
- return new ThriftFlumeEventTupleScheme();
- }
- }
-
- private static class ThriftFlumeEventTupleScheme extends TupleScheme<ThriftFlumeEvent> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, ThriftFlumeEvent struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- {
- oprot.writeI32(struct.headers.size());
- for (Map.Entry<String, String> _iter5 : struct.headers.entrySet())
- {
- oprot.writeString(_iter5.getKey());
- oprot.writeString(_iter5.getValue());
- }
- }
- oprot.writeBinary(struct.body);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, ThriftFlumeEvent struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- {
- org.apache.thrift.protocol.TMap _map6 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.headers = new HashMap<String,String>(2*_map6.size);
- String _key7;
- String _val8;
- for (int _i9 = 0; _i9 < _map6.size; ++_i9)
- {
- _key7 = iprot.readString();
- _val8 = iprot.readString();
- struct.headers.put(_key7, _val8);
- }
- }
- struct.setHeadersIsSet(true);
- struct.body = iprot.readBinary();
- struct.setBodyIsSet(true);
- }
- }
-
-}
-
diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/thrift/ThriftSourceProtocol.java b/flume-ng-sdk/src/main/java/org/apache/flume/thrift/ThriftSourceProtocol.java
deleted file mode 100644
index 72c9a2c2d..000000000
--- a/flume-ng-sdk/src/main/java/org/apache/flume/thrift/ThriftSourceProtocol.java
+++ /dev/null
@@ -1,1937 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.flume.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2017-09-01")
-public class ThriftSourceProtocol {
-
- public interface Iface {
-
- public Status append(ThriftFlumeEvent event) throws org.apache.thrift.TException;
-
- public Status appendBatch(List<ThriftFlumeEvent> events) throws org.apache.thrift.TException;
-
- }
-
- public interface AsyncIface {
-
- public void append(ThriftFlumeEvent event, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- public void appendBatch(List<ThriftFlumeEvent> events, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
- }
-
- public static class Client extends org.apache.thrift.TServiceClient implements Iface {
- public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
- public Factory() {}
- public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
- return new Client(prot);
- }
- public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
- return new Client(iprot, oprot);
- }
- }
-
- public Client(org.apache.thrift.protocol.TProtocol prot)
- {
- super(prot, prot);
- }
-
- public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
- super(iprot, oprot);
- }
-
- public Status append(ThriftFlumeEvent event) throws org.apache.thrift.TException
- {
- send_append(event);
- return recv_append();
- }
-
- public void send_append(ThriftFlumeEvent event) throws org.apache.thrift.TException
- {
- append_args args = new append_args();
- args.setEvent(event);
- sendBase("append", args);
- }
-
- public Status recv_append() throws org.apache.thrift.TException
- {
- append_result result = new append_result();
- receiveBase(result, "append");
- if (result.isSetSuccess()) {
- return result.success;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "append failed: unknown result");
- }
-
- public Status appendBatch(List<ThriftFlumeEvent> events) throws org.apache.thrift.TException
- {
- send_appendBatch(events);
- return recv_appendBatch();
- }
-
- public void send_appendBatch(List<ThriftFlumeEvent> events) throws org.apache.thrift.TException
- {
- appendBatch_args args = new appendBatch_args();
- args.setEvents(events);
- sendBase("appendBatch", args);
- }
-
- public Status recv_appendBatch() throws org.apache.thrift.TException
- {
- appendBatch_result result = new appendBatch_result();
- receiveBase(result, "appendBatch");
- if (result.isSetSuccess()) {
- return result.success;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "appendBatch failed: unknown result");
- }
-
- }
- public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
- public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
- private org.apache.thrift.async.TAsyncClientManager clientManager;
- private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
- public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
- this.clientManager = clientManager;
- this.protocolFactory = protocolFactory;
- }
- public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
- return new AsyncClient(protocolFactory, clientManager, transport);
- }
- }
-
- public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
- super(protocolFactory, clientManager, transport);
- }
-
- public void append(ThriftFlumeEvent event, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- append_call method_call = new append_call(event, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class append_call extends org.apache.thrift.async.TAsyncMethodCall {
- private ThriftFlumeEvent event;
- public append_call(ThriftFlumeEvent event, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.event = event;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("append", org.apache.thrift.protocol.TMessageType.CALL, 0));
- append_args args = new append_args();
- args.setEvent(event);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public Status getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_append();
- }
- }
-
- public void appendBatch(List<ThriftFlumeEvent> events, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
- checkReady();
- appendBatch_call method_call = new appendBatch_call(events, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class appendBatch_call extends org.apache.thrift.async.TAsyncMethodCall {
- private List<ThriftFlumeEvent> events;
- public appendBatch_call(List<ThriftFlumeEvent> events, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.events = events;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("appendBatch", org.apache.thrift.protocol.TMessageType.CALL, 0));
- appendBatch_args args = new appendBatch_args();
- args.setEvents(events);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public Status getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_appendBatch();
- }
- }
-
- }
-
- public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
- private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName());
- public Processor(I iface) {
- super(iface, getProcessMap(new HashMap<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
- }
-
- protected Processor(I iface, Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
- super(iface, getProcessMap(processMap));
- }
-
- private static <I extends Iface> Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> getProcessMap(Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
- processMap.put("append", new append());
- processMap.put("appendBatch", new appendBatch());
- return processMap;
- }
-
- public static class append<I extends Iface> extends org.apache.thrift.ProcessFunction<I, append_args> {
- public append() {
- super("append");
- }
-
- public append_args getEmptyArgsInstance() {
- return new append_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public append_result getResult(I iface, append_args args) throws org.apache.thrift.TException {
- append_result result = new append_result();
- result.success = iface.append(args.event);
- return result;
- }
- }
-
- public static class appendBatch<I extends Iface> extends org.apache.thrift.ProcessFunction<I, appendBatch_args> {
- public appendBatch() {
- super("appendBatch");
- }
-
- public appendBatch_args getEmptyArgsInstance() {
- return new appendBatch_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public appendBatch_result getResult(I iface, appendBatch_args args) throws org.apache.thrift.TException {
- appendBatch_result result = new appendBatch_result();
- result.success = iface.appendBatch(args.events);
- return result;
- }
- }
-
- }
-
- public static class AsyncProcessor<I extends AsyncIface> extends org.apache.thrift.TBaseAsyncProcessor<I> {
- private static final Logger LOGGER = LoggerFactory.getLogger(AsyncProcessor.class.getName());
- public AsyncProcessor(I iface) {
- super(iface, getProcessMap(new HashMap<String, org.apache.thrift.AsyncProcessFunction<I, ? extends org.apache.thrift.TBase, ?>>()));
- }
-
- protected AsyncProcessor(I iface, Map<String, org.apache.thrift.AsyncProcessFunction<I, ? extends org.apache.thrift.TBase, ?>> processMap) {
- super(iface, getProcessMap(processMap));
- }
-
- private static <I extends AsyncIface> Map<String, org.apache.thrift.AsyncProcessFunction<I, ? extends org.apache.thrift.TBase,?>> getProcessMap(Map<String, org.apache.thrift.AsyncProcessFunction<I, ? extends org.apache.thrift.TBase, ?>> processMap) {
- processMap.put("append", new append());
- processMap.put("appendBatch", new appendBatch());
- return processMap;
- }
-
- public static class append<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, append_args, Status> {
- public append() {
- super("append");
- }
-
- public append_args getEmptyArgsInstance() {
- return new append_args();
- }
-
- public AsyncMethodCallback<Status> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Status>() {
- public void onComplete(Status o) {
- append_result result = new append_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- append_result result = new append_result();
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, append_args args, org.apache.thrift.async.AsyncMethodCallback<Status> resultHandler) throws TException {
- iface.append(args.event,resultHandler);
- }
- }
-
- public static class appendBatch<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, appendBatch_args, Status> {
- public appendBatch() {
- super("appendBatch");
- }
-
- public appendBatch_args getEmptyArgsInstance() {
- return new appendBatch_args();
- }
-
- public AsyncMethodCallback<Status> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new AsyncMethodCallback<Status>() {
- public void onComplete(Status o) {
- appendBatch_result result = new appendBatch_result();
- result.success = o;
- try {
- fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
- return;
- } catch (Exception e) {
- LOGGER.error("Exception writing to internal frame buffer", e);
- }
- fb.close();
- }
- public void onError(Exception e) {
- byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
- org.apache.thrift.TBase msg;
- appendBatch_result result = new appendBatch_result();
- {
- msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
- msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
- }
- try {
- fcall.sendResponse(fb,msg,msgType,seqid);
- return;
- } catch (Exception ex) {
- LOGGER.error("Exception writing to internal frame buffer", ex);
- }
- fb.close();
- }
- };
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public void start(I iface, appendBatch_args args, org.apache.thrift.async.AsyncMethodCallback<Status> resultHandler) throws TException {
- iface.appendBatch(args.events,resultHandler);
- }
- }
-
- }
-
- public static class append_args implements org.apache.thrift.TBase<append_args, append_args._Fields>, java.io.Serializable, Cloneable, Comparable<append_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("append_args");
-
- private static final org.apache.thrift.protocol.TField EVENT_FIELD_DESC = new org.apache.thrift.protocol.TField("event", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new append_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new append_argsTupleSchemeFactory());
- }
-
- public ThriftFlumeEvent event; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- EVENT((short)1, "event");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // EVENT
- return EVENT;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.EVENT, new org.apache.thrift.meta_data.FieldMetaData("event", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ThriftFlumeEvent.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(append_args.class, metaDataMap);
- }
-
- public append_args() {
- }
-
- public append_args(
- ThriftFlumeEvent event)
- {
- this();
- this.event = event;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public append_args(append_args other) {
- if (other.isSetEvent()) {
- this.event = new ThriftFlumeEvent(other.event);
- }
- }
-
- public append_args deepCopy() {
- return new append_args(this);
- }
-
- @Override
- public void clear() {
- this.event = null;
- }
-
- public ThriftFlumeEvent getEvent() {
- return this.event;
- }
-
- public append_args setEvent(ThriftFlumeEvent event) {
- this.event = event;
- return this;
- }
-
- public void unsetEvent() {
- this.event = null;
- }
-
- /** Returns true if field event is set (has been assigned a value) and false otherwise */
- public boolean isSetEvent() {
- return this.event != null;
- }
-
- public void setEventIsSet(boolean value) {
- if (!value) {
- this.event = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case EVENT:
- if (value == null) {
- unsetEvent();
- } else {
- setEvent((ThriftFlumeEvent)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case EVENT:
- return getEvent();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case EVENT:
- return isSetEvent();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof append_args)
- return this.equals((append_args)that);
- return false;
- }
-
- public boolean equals(append_args that) {
- if (that == null)
- return false;
-
- boolean this_present_event = true && this.isSetEvent();
- boolean that_present_event = true && that.isSetEvent();
- if (this_present_event || that_present_event) {
- if (!(this_present_event && that_present_event))
- return false;
- if (!this.event.equals(that.event))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- List<Object> list = new ArrayList<Object>();
-
- boolean present_event = true && (isSetEvent());
- list.add(present_event);
- if (present_event)
- list.add(event);
-
- return list.hashCode();
- }
-
- @Override
- public int compareTo(append_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetEvent()).compareTo(other.isSetEvent());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetEvent()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.event, other.event);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("append_args(");
- boolean first = true;
-
- sb.append("event:");
- if (this.event == null) {
- sb.append("null");
- } else {
- sb.append(this.event);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (event != null) {
- event.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class append_argsStandardSchemeFactory implements SchemeFactory {
- public append_argsStandardScheme getScheme() {
- return new append_argsStandardScheme();
- }
- }
-
- private static class append_argsStandardScheme extends StandardScheme<append_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, append_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // EVENT
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.event = new ThriftFlumeEvent();
- struct.event.read(iprot);
- struct.setEventIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, append_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.event != null) {
- oprot.writeFieldBegin(EVENT_FIELD_DESC);
- struct.event.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class append_argsTupleSchemeFactory implements SchemeFactory {
- public append_argsTupleScheme getScheme() {
- return new append_argsTupleScheme();
- }
- }
-
- private static class append_argsTupleScheme extends TupleScheme<append_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, append_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetEvent()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetEvent()) {
- struct.event.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, append_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.event = new ThriftFlumeEvent();
- struct.event.read(iprot);
- struct.setEventIsSet(true);
- }
- }
- }
-
- }
-
- public static class append_result implements org.apache.thrift.TBase<append_result, append_result._Fields>, java.io.Serializable, Cloneable, Comparable<append_result> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("append_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.I32, (short)0);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new append_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new append_resultTupleSchemeFactory());
- }
-
- /**
- *
- * @see Status
- */
- public Status success; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- /**
- *
- * @see Status
- */
- SUCCESS((short)0, "success");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, Status.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(append_result.class, metaDataMap);
- }
-
- public append_result() {
- }
-
- public append_result(
- Status success)
- {
- this();
- this.success = success;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public append_result(append_result other) {
- if (other.isSetSuccess()) {
- this.success = other.success;
- }
- }
-
- public append_result deepCopy() {
- return new append_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- }
-
- /**
- *
- * @see Status
- */
- public Status getSuccess() {
- return this.success;
- }
-
- /**
- *
- * @see Status
- */
- public append_result setSuccess(Status success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((Status)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof append_result)
- return this.equals((append_result)that);
- return false;
- }
-
- public boolean equals(append_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- List<Object> list = new ArrayList<Object>();
-
- boolean present_success = true && (isSetSuccess());
- list.add(present_success);
- if (present_success)
- list.add(success.getValue());
-
- return list.hashCode();
- }
-
- @Override
- public int compareTo(append_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("append_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class append_resultStandardSchemeFactory implements SchemeFactory {
- public append_resultStandardScheme getScheme() {
- return new append_resultStandardScheme();
- }
- }
-
- private static class append_resultStandardScheme extends StandardScheme<append_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, append_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.success = org.apache.flume.thrift.Status.findByValue(iprot.readI32());
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
... 4870 lines suppressed ...