You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@logging.apache.org by pk...@apache.org on 2026/06/26 15:17:19 UTC
(logging-flume-rpc) branch fix/formatting updated: Format with Spotless
This is an automated email from the ASF dual-hosted git repository.
ppkarwasz pushed a commit to branch fix/formatting
in repository https://gitbox.apache.org/repos/asf/logging-flume-rpc.git
The following commit(s) were added to refs/heads/fix/formatting by this push:
new 2a571e6 Format with Spotless
2a571e6 is described below
commit 2a571e67a588fd4c932ea6af21eb0addde180a95
Author: Spotless <ro...@logging.apache.org>
AuthorDate: Fri Jun 26 17:16:16 2026 +0200
Format with Spotless
Format the project with the Spotless configuration inherited from its
parent.
No other modifications are performed.
---
.github/workflows/build.yml | 12 +-
.github/workflows/codeql-analysis.yml | 21 +-
flume-embedded-agent/pom.xml | 36 +-
.../apache/flume/agent/embedded/EmbeddedAgent.java | 388 ++-
.../agent/embedded/EmbeddedAgentConfiguration.java | 526 ++-
.../flume/agent/embedded/EmbeddedSource.java | 47 +-
.../MaterializedConfigurationProvider.java | 36 +-
.../embedded/MemoryConfigurationProvider.java | 46 +-
.../flume/agent/embedded/TestEmbeddedAgent.java | 366 +--
.../embedded/TestEmbeddedAgentConfiguration.java | 279 +-
.../embedded/TestEmbeddedAgentEmbeddedSource.java | 363 +--
.../agent/embedded/TestEmbeddedAgentState.java | 189 +-
flume-embedded-agent/src/test/resources/log4j2.xml | 33 +-
flume-rpc-avro/pom.xml | 205 +-
.../flume/rpc/avro/client/AvroCLIClient.java | 377 ++-
.../flume/rpc/avro/client/NettyAvroRpcClient.java | 1041 +++---
.../serialization/AbstractAvroEventSerializer.java | 196 +-
.../avro/serialization/AvroEventDeserializer.java | 301 +-
.../AvroEventSerializerConfigurationConstants.java | 65 +-
.../avro/serialization/DurablePositionTracker.java | 292 +-
.../FlumeEventAvroEventSerializer.java | 90 +-
.../ReliableSpoolingFileEventReader.java | 1554 +++++----
.../org/apache/flume/rpc/avro/sink/AvroSink.java | 42 +-
.../apache/flume/rpc/avro/source/AvroSource.java | 483 ++-
.../rpc/avro/source/SpoolDirectorySource.java | 541 ++-
.../flume/rpc/api/TestFailoverRpcClient.java | 268 +-
.../org/apache/flume/rpc/avro/RpcTestUtils.java | 601 ++--
.../flume/rpc/avro/TestLoadBalancingRpcClient.java | 1256 ++++---
.../flume/rpc/avro/TestRpcClientFactory.java | 222 +-
.../rpc/avro/client/TestNettyAvroRpcClient.java | 706 ++--
.../serialization/SyslogAvroEventSerializer.java | 434 ++-
.../serialization/TestAvroEventDeserializer.java | 219 +-
.../serialization/TestDurablePositionTracker.java | 107 +-
.../TestFlumeEventAvroEventSerializer.java | 181 +-
.../TestReliableSpoolingFileEventReader.java | 1424 ++++----
.../TestResettableFileInputStream.java | 1258 ++++---
.../serialization/TestSpoolingFileLineReader.java | 1099 ++++---
.../TestSyslogAvroEventSerializer.java | 178 +-
.../apache/flume/rpc/avro/sink/TestAvroSink.java | 1389 ++++----
.../rpc/avro/sink/TestDefaultSinkFactory.java | 91 +-
.../apache/flume/rpc/avro/source/MockSource.java | 78 +-
.../rpc/avro/source/TestDefaultSourceFactory.java | 109 +-
.../rpc/avro/source/TestSpoolDirectorySource.java | 993 +++---
.../node/TestAbstractConfigurationProvider.java | 593 ++--
flume-rpc-avro/src/test/resources/log4j2.xml | 33 +-
flume-rpc-thrift/pom.xml | 92 +-
.../java/org/apache/flume/rpc/thrift/Status.java | 97 +-
.../apache/flume/rpc/thrift/ThriftFlumeEvent.java | 1003 +++---
.../flume/rpc/thrift/ThriftSourceProtocol.java | 3440 ++++++++++----------
.../rpc/thrift/api/SecureRpcClientFactory.java | 53 +-
.../rpc/thrift/api/SecureThriftRpcClient.java | 193 +-
.../flume/rpc/thrift/client/ThriftRpcClient.java | 972 +++---
.../apache/flume/rpc/thrift/sink/ThriftSink.java | 58 +-
.../flume/rpc/thrift/source/ThriftSource.java | 670 ++--
.../flume/rpc/thrift/TestThriftRpcClient.java | 339 +-
.../flume/rpc/thrift/ThriftTestingSource.java | 1058 +++---
.../flume/rpc/thrift/sink/TestThriftSink.java | 560 ++--
.../apache/flume/rpc/thrift/source/MockSource.java | 78 +-
.../thrift/source/TestDefaultSourceFactory.java | 106 +-
flume-rpc-thrift/src/test/resources/log4j2.xml | 33 +-
pom.xml | 53 +-
61 files changed, 13612 insertions(+), 13961 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 284bc1a..2df41f8 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -1,17 +1,19 @@
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
+# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache license, Version 2.0
+# The ASF licenses this file to you under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
+# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the license for the specific language governing permissions and
-# limitations under the license.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
name: build
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 5b7f168..b6eb02c 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -1,13 +1,18 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
name: "CodeQL"
diff --git a/flume-embedded-agent/pom.xml b/flume-embedded-agent/pom.xml
index 42c9ac3..ce9bbe2 100644
--- a/flume-embedded-agent/pom.xml
+++ b/flume-embedded-agent/pom.xml
@@ -1,26 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to you under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <artifactId>flume-rpc</artifactId>
<groupId>org.apache.flume</groupId>
+ <artifactId>flume-rpc</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedAgent.java b/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedAgent.java
index 45c9e43..bd59e92 100644
--- a/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedAgent.java
+++ b/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedAgent.java
@@ -1,29 +1,27 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.agent.embedded;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
-
import org.apache.flume.Channel;
-import org.apache.flume.exception.ChannelException;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.FlumeException;
@@ -33,6 +31,7 @@ import org.apache.flume.SourceRunner;
import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
import org.apache.flume.conf.LogPrivacyUtil;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.lifecycle.LifecycleAware;
import org.apache.flume.lifecycle.LifecycleState;
import org.apache.flume.lifecycle.LifecycleSupervisor;
@@ -41,9 +40,6 @@ import org.apache.flume.node.MaterializedConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
/**
* EmbeddedAgent gives Flume users the ability to embed simple agents in
* applications. This Agent is mean to be much simpler than a traditional
@@ -53,207 +49,193 @@ import com.google.common.base.Preconditions;
@InterfaceAudience.Public
@InterfaceStability.Stable
public class EmbeddedAgent {
- private static final Logger LOGGER = LoggerFactory
- .getLogger(EmbeddedAgent.class);
-
- private final MaterializedConfigurationProvider configurationProvider;
- private final String name;
-
- private final LifecycleSupervisor supervisor;
- private State state;
- private SourceRunner sourceRunner;
- private Channel channel;
- private SinkRunner sinkRunner;
- private EmbeddedSource embeddedSource;
-
- @InterfaceAudience.Private
- @InterfaceStability.Unstable
- @VisibleForTesting
- EmbeddedAgent(MaterializedConfigurationProvider configurationProvider,
- String name) {
- this.configurationProvider = configurationProvider;
- this.name = name;
- state = State.NEW;
- supervisor = new LifecycleSupervisor();
-
- }
-
- public EmbeddedAgent(String name) {
- this(new MaterializedConfigurationProvider(), name);
- }
-
- /**
- * Configures the embedded agent. Can only be called after the object
- * is created or after the stop() method is called.
- *
- * @param properties source, channel, and sink group configuration
- * @throws FlumeException if a component is unable to be found or configured
- * @throws IllegalStateException if called while the agent is started
- */
- public void configure(Map<String, String> properties)
- throws FlumeException {
- if (state == State.STARTED) {
- throw new IllegalStateException("Cannot be configured while started");
+ private static final Logger LOGGER = LoggerFactory.getLogger(EmbeddedAgent.class);
+
+ private final MaterializedConfigurationProvider configurationProvider;
+ private final String name;
+
+ private final LifecycleSupervisor supervisor;
+ private State state;
+ private SourceRunner sourceRunner;
+ private Channel channel;
+ private SinkRunner sinkRunner;
+ private EmbeddedSource embeddedSource;
+
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ @VisibleForTesting
+ EmbeddedAgent(MaterializedConfigurationProvider configurationProvider, String name) {
+ this.configurationProvider = configurationProvider;
+ this.name = name;
+ state = State.NEW;
+ supervisor = new LifecycleSupervisor();
}
- doConfigure(properties);
- state = State.STOPPED;
- }
- /**
- * Started the agent. Can only be called after a successful call to
- * configure().
- *
- * @throws FlumeException if a component cannot be started
- * @throws IllegalStateException if the agent has not been configured or is
- * already started
- */
- public void start()
- throws FlumeException {
- if (state == State.STARTED) {
- throw new IllegalStateException("Cannot be started while started");
- } else if (state == State.NEW) {
- throw new IllegalStateException("Cannot be started before being " +
- "configured");
- }
- // This check needs to be done before doStart(),
- // as doStart() accesses sourceRunner.getSource()
- Source source = Preconditions.checkNotNull(sourceRunner.getSource(),
- "Source runner returned null source");
- if (source instanceof EmbeddedSource) {
- embeddedSource = (EmbeddedSource)source;
- } else {
- throw new IllegalStateException("Unknown source type: " + source.getClass().getName());
+ public EmbeddedAgent(String name) {
+ this(new MaterializedConfigurationProvider(), name);
}
- doStart();
- state = State.STARTED;
- }
- /**
- * Stops the agent. Can only be called after a successful call to start().
- * After a call to stop(), the agent can be re-configured with the
- * configure() method or re-started with the start() method.
- *
- * @throws FlumeException if a component cannot be stopped
- * @throws IllegalStateException if the agent is not started
- */
- public void stop()
- throws FlumeException {
- if (state != State.STARTED) {
- throw new IllegalStateException("Cannot be stopped unless started");
+ /**
+ * Configures the embedded agent. Can only be called after the object
+ * is created or after the stop() method is called.
+ *
+ * @param properties source, channel, and sink group configuration
+ * @throws FlumeException if a component is unable to be found or configured
+ * @throws IllegalStateException if called while the agent is started
+ */
+ public void configure(Map<String, String> properties) throws FlumeException {
+ if (state == State.STARTED) {
+ throw new IllegalStateException("Cannot be configured while started");
+ }
+ doConfigure(properties);
+ state = State.STOPPED;
}
- supervisor.stop();
- embeddedSource = null;
- state = State.STOPPED;
- }
-
- private void doConfigure(Map<String, String> properties) {
- properties = EmbeddedAgentConfiguration.configure(name, properties);
-
- if (LOGGER.isDebugEnabled() && LogPrivacyUtil.allowLogPrintConfig()) {
- LOGGER.debug("Agent configuration values");
- for (String key : new TreeSet<String>(properties.keySet())) {
- LOGGER.debug(key + " = " + properties.get(key));
- }
+ /**
+ * Started the agent. Can only be called after a successful call to
+ * configure().
+ *
+ * @throws FlumeException if a component cannot be started
+ * @throws IllegalStateException if the agent has not been configured or is
+ * already started
+ */
+ public void start() throws FlumeException {
+ if (state == State.STARTED) {
+ throw new IllegalStateException("Cannot be started while started");
+ } else if (state == State.NEW) {
+ throw new IllegalStateException("Cannot be started before being " + "configured");
+ }
+ // This check needs to be done before doStart(),
+ // as doStart() accesses sourceRunner.getSource()
+ Source source = Preconditions.checkNotNull(sourceRunner.getSource(), "Source runner returned null source");
+ if (source instanceof EmbeddedSource) {
+ embeddedSource = (EmbeddedSource) source;
+ } else {
+ throw new IllegalStateException(
+ "Unknown source type: " + source.getClass().getName());
+ }
+ doStart();
+ state = State.STARTED;
}
- MaterializedConfiguration conf = configurationProvider.get(name,
- properties);
- Map<String, SourceRunner> sources = conf.getSourceRunners();
- if (sources.size() != 1) {
- throw new FlumeException("Expected one source and got " +
- sources.size());
- }
- Map<String, Channel> channels = conf.getChannels();
- if (channels.size() != 1) {
- throw new FlumeException("Expected one channel and got " +
- channels.size());
- }
- Map<String, SinkRunner> sinks = conf.getSinkRunners();
- if (sinks.size() != 1) {
- throw new FlumeException("Expected one sink group and got " +
- sinks.size());
+ /**
+ * Stops the agent. Can only be called after a successful call to start().
+ * After a call to stop(), the agent can be re-configured with the
+ * configure() method or re-started with the start() method.
+ *
+ * @throws FlumeException if a component cannot be stopped
+ * @throws IllegalStateException if the agent is not started
+ */
+ public void stop() throws FlumeException {
+ if (state != State.STARTED) {
+ throw new IllegalStateException("Cannot be stopped unless started");
+ }
+ supervisor.stop();
+ embeddedSource = null;
+ state = State.STOPPED;
}
- this.sourceRunner = sources.values().iterator().next();
- this.channel = channels.values().iterator().next();
- this.sinkRunner = sinks.values().iterator().next();
- }
- /**
- * Adds event to the channel owned by the agent. Note however, that the
- * event is not copied and as such, the byte array and headers cannot
- * be re-used by the caller.
- * @param event
- * @throws EventDeliveryException if unable to add event to channel
- */
- public void put(Event event) throws EventDeliveryException {
- if (state != State.STARTED) {
- throw new IllegalStateException("Cannot put events unless started");
- }
- try {
- embeddedSource.put(event);
- } catch (ChannelException ex) {
- throw new EventDeliveryException("Embedded agent " + name +
- ": Unable to process event: " + ex.getMessage(), ex);
+ private void doConfigure(Map<String, String> properties) {
+
+ properties = EmbeddedAgentConfiguration.configure(name, properties);
+
+ if (LOGGER.isDebugEnabled() && LogPrivacyUtil.allowLogPrintConfig()) {
+ LOGGER.debug("Agent configuration values");
+ for (String key : new TreeSet<String>(properties.keySet())) {
+ LOGGER.debug(key + " = " + properties.get(key));
+ }
+ }
+
+ MaterializedConfiguration conf = configurationProvider.get(name, properties);
+ Map<String, SourceRunner> sources = conf.getSourceRunners();
+ if (sources.size() != 1) {
+ throw new FlumeException("Expected one source and got " + sources.size());
+ }
+ Map<String, Channel> channels = conf.getChannels();
+ if (channels.size() != 1) {
+ throw new FlumeException("Expected one channel and got " + channels.size());
+ }
+ Map<String, SinkRunner> sinks = conf.getSinkRunners();
+ if (sinks.size() != 1) {
+ throw new FlumeException("Expected one sink group and got " + sinks.size());
+ }
+ this.sourceRunner = sources.values().iterator().next();
+ this.channel = channels.values().iterator().next();
+ this.sinkRunner = sinks.values().iterator().next();
}
- }
- /**
- * Adds events to the channel owned by the agent. Note however, that the
- * event is not copied and as such, the byte array and headers cannot
- * be re-used by the caller.
- * @param events
- * @throws EventDeliveryException if unable to add event to channel
- */
- public void putAll(List<Event> events) throws EventDeliveryException {
- if (state != State.STARTED) {
- throw new IllegalStateException("Cannot put events unless started");
- }
- try {
- embeddedSource.putAll(events);
- } catch (ChannelException ex) {
- throw new EventDeliveryException("Embedded agent " + name +
- ": Unable to process event: " + ex.getMessage(), ex);
+ /**
+ * Adds event to the channel owned by the agent. Note however, that the
+ * event is not copied and as such, the byte array and headers cannot
+ * be re-used by the caller.
+ * @param event
+ * @throws EventDeliveryException if unable to add event to channel
+ */
+ public void put(Event event) throws EventDeliveryException {
+ if (state != State.STARTED) {
+ throw new IllegalStateException("Cannot put events unless started");
+ }
+ try {
+ embeddedSource.put(event);
+ } catch (ChannelException ex) {
+ throw new EventDeliveryException(
+ "Embedded agent " + name + ": Unable to process event: " + ex.getMessage(), ex);
+ }
}
- }
- private void doStart() {
- boolean error = true;
- try {
- channel.start();
- sinkRunner.start();
- sourceRunner.start();
+ /**
+ * Adds events to the channel owned by the agent. Note however, that the
+ * event is not copied and as such, the byte array and headers cannot
+ * be re-used by the caller.
+ * @param events
+ * @throws EventDeliveryException if unable to add event to channel
+ */
+ public void putAll(List<Event> events) throws EventDeliveryException {
+ if (state != State.STARTED) {
+ throw new IllegalStateException("Cannot put events unless started");
+ }
+ try {
+ embeddedSource.putAll(events);
+ } catch (ChannelException ex) {
+ throw new EventDeliveryException(
+ "Embedded agent " + name + ": Unable to process event: " + ex.getMessage(), ex);
+ }
+ }
- supervisor.supervise(channel,
- new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
- supervisor.supervise(sinkRunner,
- new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
- supervisor.supervise(sourceRunner,
- new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
- error = false;
- } finally {
- if (error) {
- stopLogError(sourceRunner);
- stopLogError(channel);
- stopLogError(sinkRunner);
- supervisor.stop();
- }
+ private void doStart() {
+ boolean error = true;
+ try {
+ channel.start();
+ sinkRunner.start();
+ sourceRunner.start();
+
+ supervisor.supervise(channel, new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
+ supervisor.supervise(sinkRunner, new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
+ supervisor.supervise(sourceRunner, new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
+ error = false;
+ } finally {
+ if (error) {
+ stopLogError(sourceRunner);
+ stopLogError(channel);
+ stopLogError(sinkRunner);
+ supervisor.stop();
+ }
+ }
}
- }
- private void stopLogError(LifecycleAware lifeCycleAware) {
- try {
- if (LifecycleState.START.equals(lifeCycleAware.getLifecycleState())) {
- lifeCycleAware.stop();
- }
- } catch (Exception e) {
- LOGGER.warn("Exception while stopping " + lifeCycleAware, e);
+ private void stopLogError(LifecycleAware lifeCycleAware) {
+ try {
+ if (LifecycleState.START.equals(lifeCycleAware.getLifecycleState())) {
+ lifeCycleAware.stop();
+ }
+ } catch (Exception e) {
+ LOGGER.warn("Exception while stopping " + lifeCycleAware, e);
+ }
}
- }
- private static enum State {
- NEW(),
- STOPPED(),
- STARTED();
- }
+ private static enum State {
+ NEW(),
+ STOPPED(),
+ STARTED();
+ }
}
diff --git a/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedAgentConfiguration.java b/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedAgentConfiguration.java
index fe728f9..9ff4cce 100644
--- a/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedAgentConfiguration.java
+++ b/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedAgentConfiguration.java
@@ -1,30 +1,30 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.agent.embedded;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
-
import org.apache.flume.FlumeException;
import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
@@ -33,10 +33,6 @@ import org.apache.flume.conf.channel.ChannelType;
import org.apache.flume.conf.sink.SinkProcessorType;
import org.apache.flume.conf.sink.SinkType;
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Maps;
-
/**
* Stores publicly accessible configuration constants and private
* configuration constants and methods.
@@ -44,280 +40,270 @@ import com.google.common.collect.Maps;
@InterfaceAudience.Public
@InterfaceStability.Stable
public class EmbeddedAgentConfiguration {
- public static final String SEPERATOR = ".";
- private static final Joiner JOINER = Joiner.on(SEPERATOR);
- private static final String TYPE = "type";
+ public static final String SEPERATOR = ".";
+ private static final Joiner JOINER = Joiner.on(SEPERATOR);
+ private static final String TYPE = "type";
- /**
- * Prefix for source properties
- */
- public static final String SOURCE = "source";
- /**
- * Prefix for channel properties
- */
- public static final String CHANNEL = "channel";
- /**
- * Prefix for sink processor properties
- */
- public static final String SINK_PROCESSOR = "processor";
- /**
- * Space delimited list of sink names: e.g. sink1 sink2 sink3
- */
- public static final String SINKS = "sinks";
+ /**
+ * Prefix for source properties
+ */
+ public static final String SOURCE = "source";
+ /**
+ * Prefix for channel properties
+ */
+ public static final String CHANNEL = "channel";
+ /**
+ * Prefix for sink processor properties
+ */
+ public static final String SINK_PROCESSOR = "processor";
+ /**
+ * Space delimited list of sink names: e.g. sink1 sink2 sink3
+ */
+ public static final String SINKS = "sinks";
- public static final String SINKS_PREFIX = join(SINKS, "");
- /**
- * Source type, choices are `embedded'
- */
- public static final String SOURCE_TYPE = join(SOURCE, TYPE);
- /**
- * Prefix for passing configuration parameters to the source
- */
- public static final String SOURCE_PREFIX = join(SOURCE, "");
- /**
- * Channel type, choices are `memory' or `file'
- */
- public static final String CHANNEL_TYPE = join(CHANNEL, TYPE);
- /**
- * Prefix for passing configuration parameters to the channel
- */
- public static final String CHANNEL_PREFIX = join(CHANNEL, "");
- /**
- * Sink processor type, choices are `default', `failover' or `load_balance'
- */
- public static final String SINK_PROCESSOR_TYPE = join(SINK_PROCESSOR, TYPE);
- /**
- * Prefix for passing configuration parameters to the sink processor
- */
- public static final String SINK_PROCESSOR_PREFIX = join(SINK_PROCESSOR, "");
- /**
- * Embedded source which provides simple in-memory transfer to channel.
- * Use this source via the put,putAll methods on the EmbeddedAgent. This
- * is the only supported source to use for Embedded Agents.
- */
- public static final String SOURCE_TYPE_EMBEDDED = EmbeddedSource.class.getName();
- private static final String SOURCE_TYPE_EMBEDDED_ALIAS = "EMBEDDED";
- /**
- * Memory channel which stores events in heap. See Flume User Guide for
- * configuration information. This is the recommended channel to use for
- * Embedded Agents.
- */
- public static final String CHANNEL_TYPE_MEMORY = ChannelType.MEMORY.name();
+ public static final String SINKS_PREFIX = join(SINKS, "");
+ /**
+ * Source type, choices are `embedded'
+ */
+ public static final String SOURCE_TYPE = join(SOURCE, TYPE);
+ /**
+ * Prefix for passing configuration parameters to the source
+ */
+ public static final String SOURCE_PREFIX = join(SOURCE, "");
+ /**
+ * Channel type, choices are `memory' or `file'
+ */
+ public static final String CHANNEL_TYPE = join(CHANNEL, TYPE);
+ /**
+ * Prefix for passing configuration parameters to the channel
+ */
+ public static final String CHANNEL_PREFIX = join(CHANNEL, "");
+ /**
+ * Sink processor type, choices are `default', `failover' or `load_balance'
+ */
+ public static final String SINK_PROCESSOR_TYPE = join(SINK_PROCESSOR, TYPE);
+ /**
+ * Prefix for passing configuration parameters to the sink processor
+ */
+ public static final String SINK_PROCESSOR_PREFIX = join(SINK_PROCESSOR, "");
+ /**
+ * Embedded source which provides simple in-memory transfer to channel.
+ * Use this source via the put,putAll methods on the EmbeddedAgent. This
+ * is the only supported source to use for Embedded Agents.
+ */
+ public static final String SOURCE_TYPE_EMBEDDED = EmbeddedSource.class.getName();
- /**
- * Spillable Memory channel which stores events in heap. See Flume User Guide for
- * configuration information. This is the recommended channel to use for
- * Embedded Agents.
- */
- public static final String CHANNEL_TYPE_SPILLABLEMEMORY = ChannelType.SPILLABLEMEMORY.name();
+ private static final String SOURCE_TYPE_EMBEDDED_ALIAS = "EMBEDDED";
+ /**
+ * Memory channel which stores events in heap. See Flume User Guide for
+ * configuration information. This is the recommended channel to use for
+ * Embedded Agents.
+ */
+ public static final String CHANNEL_TYPE_MEMORY = ChannelType.MEMORY.name();
- /**
- * File based channel which stores events in on local disk. See Flume User
- * Guide for configuration information.
- */
- public static final String CHANNEL_TYPE_FILE = ChannelType.FILE.name();
+ /**
+ * Spillable Memory channel which stores events in heap. See Flume User Guide for
+ * configuration information. This is the recommended channel to use for
+ * Embedded Agents.
+ */
+ public static final String CHANNEL_TYPE_SPILLABLEMEMORY = ChannelType.SPILLABLEMEMORY.name();
+
+ /**
+ * File based channel which stores events in on local disk. See Flume User
+ * Guide for configuration information.
+ */
+ public static final String CHANNEL_TYPE_FILE = ChannelType.FILE.name();
- /**
- * Avro sink which can send events to a downstream avro source. This is the
- * only supported sink for Embedded Agents.
- */
- public static final String SINK_TYPE_AVRO = SinkType.AVRO.name();
+ /**
+ * Avro sink which can send events to a downstream avro source. This is the
+ * only supported sink for Embedded Agents.
+ */
+ public static final String SINK_TYPE_AVRO = SinkType.AVRO.name();
- /**
- * Default sink processors which may be used when there is only a single sink.
- */
- public static final String SINK_PROCESSOR_TYPE_DEFAULT = SinkProcessorType.DEFAULT.name();
- /**
- * Failover sink processor. See Flume User Guide for configuration
- * information.
- */
- public static final String SINK_PROCESSOR_TYPE_FAILOVER = SinkProcessorType.FAILOVER.name();
- /**
- * Load balancing sink processor. See Flume User Guide for configuration
- * information.
- */
- public static final String SINK_PROCESSOR_TYPE_LOAD_BALANCE =
- SinkProcessorType.LOAD_BALANCE.name();
+ /**
+ * Default sink processors which may be used when there is only a single sink.
+ */
+ public static final String SINK_PROCESSOR_TYPE_DEFAULT = SinkProcessorType.DEFAULT.name();
+ /**
+ * Failover sink processor. See Flume User Guide for configuration
+ * information.
+ */
+ public static final String SINK_PROCESSOR_TYPE_FAILOVER = SinkProcessorType.FAILOVER.name();
+ /**
+ * Load balancing sink processor. See Flume User Guide for configuration
+ * information.
+ */
+ public static final String SINK_PROCESSOR_TYPE_LOAD_BALANCE = SinkProcessorType.LOAD_BALANCE.name();
- private static final String[] ALLOWED_SOURCES = {
- SOURCE_TYPE_EMBEDDED_ALIAS,
- SOURCE_TYPE_EMBEDDED,
- };
+ private static final String[] ALLOWED_SOURCES = {
+ SOURCE_TYPE_EMBEDDED_ALIAS, SOURCE_TYPE_EMBEDDED,
+ };
- private static final String[] ALLOWED_CHANNELS = {
- CHANNEL_TYPE_MEMORY,
- CHANNEL_TYPE_FILE
- };
+ private static final String[] ALLOWED_CHANNELS = {CHANNEL_TYPE_MEMORY, CHANNEL_TYPE_FILE};
- private static final String[] ALLOWED_SINKS = {
- SINK_TYPE_AVRO
- };
+ private static final String[] ALLOWED_SINKS = {SINK_TYPE_AVRO};
- private static final String[] ALLOWED_SINK_PROCESSORS = {
- SINK_PROCESSOR_TYPE_DEFAULT,
- SINK_PROCESSOR_TYPE_FAILOVER,
- SINK_PROCESSOR_TYPE_LOAD_BALANCE
- };
+ private static final String[] ALLOWED_SINK_PROCESSORS = {
+ SINK_PROCESSOR_TYPE_DEFAULT, SINK_PROCESSOR_TYPE_FAILOVER, SINK_PROCESSOR_TYPE_LOAD_BALANCE
+ };
- private static final ImmutableList<String> DISALLOWED_SINK_NAMES =
- ImmutableList.of("source", "channel", "processor");
+ private static final ImmutableList<String> DISALLOWED_SINK_NAMES =
+ ImmutableList.of("source", "channel", "processor");
- private static void validate(String name,
- Map<String, String> properties) throws FlumeException {
+ private static void validate(String name, Map<String, String> properties) throws FlumeException {
- if (properties.containsKey(SOURCE_TYPE)) {
- checkAllowed(ALLOWED_SOURCES, properties.get(SOURCE_TYPE));
- }
- checkRequired(properties, CHANNEL_TYPE);
- checkAllowed(ALLOWED_CHANNELS, properties.get(CHANNEL_TYPE));
- checkRequired(properties, SINKS);
- String sinkNames = properties.get(SINKS);
- for (String sink : sinkNames.split("\\s+")) {
- if (DISALLOWED_SINK_NAMES.contains(sink.toLowerCase(Locale.ENGLISH))) {
- throw new FlumeException("Sink name " + sink + " is one of the" +
- " disallowed sink names: " + DISALLOWED_SINK_NAMES);
- }
- String key = join(sink, TYPE);
- checkRequired(properties, key);
- checkAllowed(ALLOWED_SINKS, properties.get(key));
+ if (properties.containsKey(SOURCE_TYPE)) {
+ checkAllowed(ALLOWED_SOURCES, properties.get(SOURCE_TYPE));
+ }
+ checkRequired(properties, CHANNEL_TYPE);
+ checkAllowed(ALLOWED_CHANNELS, properties.get(CHANNEL_TYPE));
+ checkRequired(properties, SINKS);
+ String sinkNames = properties.get(SINKS);
+ for (String sink : sinkNames.split("\\s+")) {
+ if (DISALLOWED_SINK_NAMES.contains(sink.toLowerCase(Locale.ENGLISH))) {
+ throw new FlumeException(
+ "Sink name " + sink + " is one of the" + " disallowed sink names: " + DISALLOWED_SINK_NAMES);
+ }
+ String key = join(sink, TYPE);
+ checkRequired(properties, key);
+ checkAllowed(ALLOWED_SINKS, properties.get(key));
+ }
+ checkRequired(properties, SINK_PROCESSOR_TYPE);
+ checkAllowed(ALLOWED_SINK_PROCESSORS, properties.get(SINK_PROCESSOR_TYPE));
}
- checkRequired(properties, SINK_PROCESSOR_TYPE);
- checkAllowed(ALLOWED_SINK_PROCESSORS, properties.get(SINK_PROCESSOR_TYPE));
- }
- /**
- * Folds embedded configuration structure into an agent configuration.
- * Should only be called after validate returns without error.
- *
- * @param name - agent name
- * @param properties - embedded agent configuration
- * @return configuration applicable to a flume agent
- */
- @InterfaceAudience.Private
- @InterfaceStability.Unstable
- static Map<String, String> configure(String name,
- Map<String, String> properties) throws FlumeException {
- validate(name, properties);
- // we are going to modify the properties as we parse the config
- properties = new HashMap<String, String>(properties);
+ /**
+ * Folds embedded configuration structure into an agent configuration.
+ * Should only be called after validate returns without error.
+ *
+ * @param name - agent name
+ * @param properties - embedded agent configuration
+ * @return configuration applicable to a flume agent
+ */
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ static Map<String, String> configure(String name, Map<String, String> properties) throws FlumeException {
+ validate(name, properties);
+ // we are going to modify the properties as we parse the config
+ properties = new HashMap<String, String>(properties);
- if (!properties.containsKey(SOURCE_TYPE) ||
- SOURCE_TYPE_EMBEDDED_ALIAS.equalsIgnoreCase(properties.get(SOURCE_TYPE))) {
- properties.put(SOURCE_TYPE, SOURCE_TYPE_EMBEDDED);
- }
- String sinkNames = properties.remove(SINKS);
+ if (!properties.containsKey(SOURCE_TYPE)
+ || SOURCE_TYPE_EMBEDDED_ALIAS.equalsIgnoreCase(properties.get(SOURCE_TYPE))) {
+ properties.put(SOURCE_TYPE, SOURCE_TYPE_EMBEDDED);
+ }
+ String sinkNames = properties.remove(SINKS);
- String strippedName = name.replaceAll("\\s+","");
- String sourceName = "source-" + strippedName;
- String channelName = "channel-" + strippedName;
- String sinkGroupName = "sink-group-" + strippedName;
+ String strippedName = name.replaceAll("\\s+", "");
+ String sourceName = "source-" + strippedName;
+ String channelName = "channel-" + strippedName;
+ String sinkGroupName = "sink-group-" + strippedName;
- /*
- * Now we are going to process the user supplied configuration
- * and generate an agent configuration. This is only to supply
- * a simpler client api than passing in an entire agent configuration.
- */
- // user supplied config -> agent configuration
- Map<String, String> result = Maps.newHashMap();
+ /*
+ * Now we are going to process the user supplied configuration
+ * and generate an agent configuration. This is only to supply
+ * a simpler client api than passing in an entire agent configuration.
+ */
+ // user supplied config -> agent configuration
+ Map<String, String> result = Maps.newHashMap();
- /*
- * First we are going to setup all the root level pointers. I.E
- * point the agent at the components, sink group at sinks, and
- * source at the channel.
- */
- // point agent at source
- result.put(join(name, BasicConfigurationConstants.CONFIG_SOURCES),
- sourceName);
- // point agent at channel
- result.put(join(name, BasicConfigurationConstants.CONFIG_CHANNELS),
- channelName);
- // point agent at sinks
- result.put(join(name, BasicConfigurationConstants.CONFIG_SINKS),
- sinkNames);
- // points the agent at the sinkgroup
- result.put(join(name, BasicConfigurationConstants.CONFIG_SINKGROUPS),
- sinkGroupName);
- // points the sinkgroup at the sinks
- result.put(join(name, BasicConfigurationConstants.CONFIG_SINKGROUPS,
- sinkGroupName, SINKS), sinkNames);
- // points the source at the channel
- result.put(join(name,
- BasicConfigurationConstants.CONFIG_SOURCES, sourceName,
- BasicConfigurationConstants.CONFIG_CHANNELS), channelName);
+ /*
+ * First we are going to setup all the root level pointers. I.E
+ * point the agent at the components, sink group at sinks, and
+ * source at the channel.
+ */
+ // point agent at source
+ result.put(join(name, BasicConfigurationConstants.CONFIG_SOURCES), sourceName);
+ // point agent at channel
+ result.put(join(name, BasicConfigurationConstants.CONFIG_CHANNELS), channelName);
+ // point agent at sinks
+ result.put(join(name, BasicConfigurationConstants.CONFIG_SINKS), sinkNames);
+ // points the agent at the sinkgroup
+ result.put(join(name, BasicConfigurationConstants.CONFIG_SINKGROUPS), sinkGroupName);
+ // points the sinkgroup at the sinks
+ result.put(join(name, BasicConfigurationConstants.CONFIG_SINKGROUPS, sinkGroupName, SINKS), sinkNames);
+ // points the source at the channel
+ result.put(
+ join(
+ name,
+ BasicConfigurationConstants.CONFIG_SOURCES,
+ sourceName,
+ BasicConfigurationConstants.CONFIG_CHANNELS),
+ channelName);
- // Properties will be modified during iteration so we need a
- // copy of the keys.
- Set<String> userProvidedKeys = new HashSet<String>(properties.keySet());
+ // Properties will be modified during iteration so we need a
+ // copy of the keys.
+ Set<String> userProvidedKeys = new HashSet<String>(properties.keySet());
- /*
- * Second process the sink configuration and point the sinks
- * at the channel.
- */
- for (String sink : sinkNames.split("\\s+")) {
- for (String key : userProvidedKeys) {
- String value = properties.get(key);
- if (key.startsWith(sink + SEPERATOR)) {
- properties.remove(key);
- result.put(join(name,
- BasicConfigurationConstants.CONFIG_SINKS, key), value);
+ /*
+ * Second process the sink configuration and point the sinks
+ * at the channel.
+ */
+ for (String sink : sinkNames.split("\\s+")) {
+ for (String key : userProvidedKeys) {
+ String value = properties.get(key);
+ if (key.startsWith(sink + SEPERATOR)) {
+ properties.remove(key);
+ result.put(join(name, BasicConfigurationConstants.CONFIG_SINKS, key), value);
+ }
+ }
+ // point the sink at the channel
+ result.put(
+ join(
+ name,
+ BasicConfigurationConstants.CONFIG_SINKS,
+ sink,
+ BasicConfigurationConstants.CONFIG_CHANNEL),
+ channelName);
}
- }
- // point the sink at the channel
- result.put(join(name,
- BasicConfigurationConstants.CONFIG_SINKS, sink,
- BasicConfigurationConstants.CONFIG_CHANNEL), channelName);
- }
- /*
- * Third, process all remaining configuration items, prefixing them
- * correctly and then passing them on to the agent.
- */
- userProvidedKeys = new HashSet<String>(properties.keySet());
- for (String key : userProvidedKeys) {
- String value = properties.get(key);
- if (key.startsWith(SOURCE_PREFIX)) {
- // users use `source' but agent needs the actual source name
- key = key.replaceFirst(SOURCE, sourceName);
- result.put(join(name,
- BasicConfigurationConstants.CONFIG_SOURCES, key), value);
- } else if (key.startsWith(CHANNEL_PREFIX)) {
- // users use `channel' but agent needs the actual channel name
- key = key.replaceFirst(CHANNEL, channelName);
- result.put(join(name,
- BasicConfigurationConstants.CONFIG_CHANNELS, key), value);
- } else if (key.startsWith(SINK_PROCESSOR_PREFIX)) {
- // agent.sinkgroups.sinkgroup.processor.*
- result.put(join(name, BasicConfigurationConstants.CONFIG_SINKGROUPS,
- sinkGroupName, key), value);
- } else {
- // XXX should we simply ignore this?
- throw new FlumeException("Unknown configuration " + key);
- }
- }
- return result;
- }
- private static void checkAllowed(String[] allowedTypes, String type) {
- boolean isAllowed = false;
- type = type.trim();
- for (String allowedType : allowedTypes) {
- if (allowedType.equalsIgnoreCase(type)) {
- isAllowed = true;
- break;
- }
+ /*
+ * Third, process all remaining configuration items, prefixing them
+ * correctly and then passing them on to the agent.
+ */
+ userProvidedKeys = new HashSet<String>(properties.keySet());
+ for (String key : userProvidedKeys) {
+ String value = properties.get(key);
+ if (key.startsWith(SOURCE_PREFIX)) {
+ // users use `source' but agent needs the actual source name
+ key = key.replaceFirst(SOURCE, sourceName);
+ result.put(join(name, BasicConfigurationConstants.CONFIG_SOURCES, key), value);
+ } else if (key.startsWith(CHANNEL_PREFIX)) {
+ // users use `channel' but agent needs the actual channel name
+ key = key.replaceFirst(CHANNEL, channelName);
+ result.put(join(name, BasicConfigurationConstants.CONFIG_CHANNELS, key), value);
+ } else if (key.startsWith(SINK_PROCESSOR_PREFIX)) {
+ // agent.sinkgroups.sinkgroup.processor.*
+ result.put(join(name, BasicConfigurationConstants.CONFIG_SINKGROUPS, sinkGroupName, key), value);
+ } else {
+ // XXX should we simply ignore this?
+ throw new FlumeException("Unknown configuration " + key);
+ }
+ }
+ return result;
}
- if (!isAllowed) {
- throw new FlumeException("Component type of " + type + " is not in " +
- "allowed types of " + Arrays.toString(allowedTypes));
+
+ private static void checkAllowed(String[] allowedTypes, String type) {
+ boolean isAllowed = false;
+ type = type.trim();
+ for (String allowedType : allowedTypes) {
+ if (allowedType.equalsIgnoreCase(type)) {
+ isAllowed = true;
+ break;
+ }
+ }
+ if (!isAllowed) {
+ throw new FlumeException(
+ "Component type of " + type + " is not in " + "allowed types of " + Arrays.toString(allowedTypes));
+ }
}
- }
- private static void checkRequired(Map<String, String> properties, String name) {
- if (!properties.containsKey(name)) {
- throw new FlumeException("Required parameter not found " + name);
+
+ private static void checkRequired(Map<String, String> properties, String name) {
+ if (!properties.containsKey(name)) {
+ throw new FlumeException("Required parameter not found " + name);
+ }
}
- }
- private static String join(String... parts) {
- return JOINER.join(parts);
- }
+ private static String join(String... parts) {
+ return JOINER.join(parts);
+ }
- private EmbeddedAgentConfiguration() {
- }
-}
\ No newline at end of file
+ private EmbeddedAgentConfiguration() {}
+}
diff --git a/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedSource.java b/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedSource.java
index 375e27b..5931e0a 100644
--- a/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedSource.java
+++ b/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/EmbeddedSource.java
@@ -1,32 +1,29 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.agent.embedded;
import java.util.List;
-
-import org.apache.flume.exception.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDrivenSource;
import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
import org.apache.flume.conf.Configurable;
+import org.apache.flume.exception.ChannelException;
import org.apache.flume.source.AbstractSource;
/**
@@ -37,16 +34,14 @@ import org.apache.flume.source.AbstractSource;
@InterfaceStability.Unstable
public class EmbeddedSource extends AbstractSource implements EventDrivenSource, Configurable {
- @Override
- public void configure(Context context) {
- }
-
- public void put(Event event) throws ChannelException {
- getChannelProcessor().processEvent(event);
- }
+ @Override
+ public void configure(Context context) {}
- public void putAll(List<Event> events) throws ChannelException {
- getChannelProcessor().processEventBatch(events);
- }
+ public void put(Event event) throws ChannelException {
+ getChannelProcessor().processEvent(event);
+ }
+ public void putAll(List<Event> events) throws ChannelException {
+ getChannelProcessor().processEventBatch(events);
+ }
}
diff --git a/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/MaterializedConfigurationProvider.java b/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/MaterializedConfigurationProvider.java
index 9d93138..69f3874 100644
--- a/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/MaterializedConfigurationProvider.java
+++ b/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/MaterializedConfigurationProvider.java
@@ -1,25 +1,22 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.agent.embedded;
import java.util.Map;
-
import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
import org.apache.flume.node.MaterializedConfiguration;
@@ -34,9 +31,8 @@ import org.apache.flume.node.MaterializedConfiguration;
@InterfaceStability.Unstable
class MaterializedConfigurationProvider {
- MaterializedConfiguration get(String name, Map<String, String> properties) {
- MemoryConfigurationProvider confProvider =
- new MemoryConfigurationProvider(name, properties);
- return confProvider.getConfiguration();
- }
+ MaterializedConfiguration get(String name, Map<String, String> properties) {
+ MemoryConfigurationProvider confProvider = new MemoryConfigurationProvider(name, properties);
+ return confProvider.getConfiguration();
+ }
}
diff --git a/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/MemoryConfigurationProvider.java b/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/MemoryConfigurationProvider.java
index 0377501..898ff18 100644
--- a/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/MemoryConfigurationProvider.java
+++ b/flume-embedded-agent/src/main/java/org/apache/flume/agent/embedded/MemoryConfigurationProvider.java
@@ -1,25 +1,22 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.agent.embedded;
import java.util.Map;
-
import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
import org.apache.flume.conf.FlumeConfiguration;
@@ -33,16 +30,15 @@ import org.apache.flume.node.AbstractConfigurationProvider;
@InterfaceAudience.Private
@InterfaceStability.Unstable
class MemoryConfigurationProvider extends AbstractConfigurationProvider {
- private final Map<String, String> properties;
-
- MemoryConfigurationProvider(String name, Map<String, String> properties) {
- super(name);
- this.properties = properties;
- }
+ private final Map<String, String> properties;
- @Override
- protected FlumeConfiguration getFlumeConfiguration() {
- return new FlumeConfiguration(properties);
- }
+ MemoryConfigurationProvider(String name, Map<String, String> properties) {
+ super(name);
+ this.properties = properties;
+ }
+ @Override
+ protected FlumeConfiguration getFlumeConfiguration() {
+ return new FlumeConfiguration(properties);
+ }
}
diff --git a/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgent.java b/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgent.java
index 68a3a07..4193bf5 100644
--- a/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgent.java
+++ b/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgent.java
@@ -1,23 +1,25 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.agent.embedded;
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
@@ -27,9 +29,8 @@ import java.util.Map;
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.avro.ipc.netty.NettyServer;
import org.apache.avro.ipc.Responder;
+import org.apache.avro.ipc.netty.NettyServer;
import org.apache.avro.ipc.specific.SpecificResponder;
import org.apache.flume.Event;
import org.apache.flume.event.EventBuilder;
@@ -43,188 +44,179 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
public class TestEmbeddedAgent {
- private static final Logger LOGGER = LoggerFactory
- .getLogger(TestEmbeddedAgent.class);
- private static final String HOSTNAME = "localhost";
- private static AtomicInteger serialNumber = new AtomicInteger(0);
- private EmbeddedAgent agent;
- private Map<String, String> properties;
- private EventCollector eventCollector;
- private NettyServer nettyServer;
- private Map<String, String> headers;
- private byte[] body;
-
- @Before
- public void setUp() throws Exception {
- headers = Maps.newHashMap();
- headers.put("key1", "value1");
- body = "body".getBytes(Charsets.UTF_8);
-
- int port = findFreePort();
- eventCollector = new EventCollector();
- Responder responder = new SpecificResponder(AvroSourceProtocol.class,
- eventCollector);
- nettyServer = new NettyServer(responder,
- new InetSocketAddress(HOSTNAME, port));
- nettyServer.start();
-
- // give the server a second to start
- Thread.sleep(1000L);
-
- properties = Maps.newHashMap();
- properties.put("channel.type", "memory");
- properties.put("channel.capacity", "200");
- properties.put("sinks", "sink1 sink2");
- properties.put("sink1.type", "avro");
- properties.put("sink2.type", "avro");
- properties.put("sink1.hostname", HOSTNAME);
- properties.put("sink1.port", String.valueOf(port));
- properties.put("sink2.hostname", HOSTNAME);
- properties.put("sink2.port", String.valueOf(port));
- properties.put("processor.type", "load_balance");
-
- agent = new EmbeddedAgent("test-" + serialNumber.incrementAndGet());
- }
-
- @After
- public void tearDown() throws Exception {
- if (agent != null) {
- try {
- agent.stop();
- } catch (Exception e) {
- LOGGER.debug("Error shutting down agent", e);
- }
- }
- if (nettyServer != null) {
- try {
- nettyServer.close();
- } catch (Exception e) {
- LOGGER.debug("Error shutting down server", e);
- }
+ private static final Logger LOGGER = LoggerFactory.getLogger(TestEmbeddedAgent.class);
+ private static final String HOSTNAME = "localhost";
+ private static AtomicInteger serialNumber = new AtomicInteger(0);
+ private EmbeddedAgent agent;
+ private Map<String, String> properties;
+ private EventCollector eventCollector;
+ private NettyServer nettyServer;
+ private Map<String, String> headers;
+ private byte[] body;
+
+ @Before
+ public void setUp() throws Exception {
+ headers = Maps.newHashMap();
+ headers.put("key1", "value1");
+ body = "body".getBytes(Charsets.UTF_8);
+
+ int port = findFreePort();
+ eventCollector = new EventCollector();
+ Responder responder = new SpecificResponder(AvroSourceProtocol.class, eventCollector);
+ nettyServer = new NettyServer(responder, new InetSocketAddress(HOSTNAME, port));
+ nettyServer.start();
+
+ // give the server a second to start
+ Thread.sleep(1000L);
+
+ properties = Maps.newHashMap();
+ properties.put("channel.type", "memory");
+ properties.put("channel.capacity", "200");
+ properties.put("sinks", "sink1 sink2");
+ properties.put("sink1.type", "avro");
+ properties.put("sink2.type", "avro");
+ properties.put("sink1.hostname", HOSTNAME);
+ properties.put("sink1.port", String.valueOf(port));
+ properties.put("sink2.hostname", HOSTNAME);
+ properties.put("sink2.port", String.valueOf(port));
+ properties.put("processor.type", "load_balance");
+
+ agent = new EmbeddedAgent("test-" + serialNumber.incrementAndGet());
}
- }
-
- @Test(timeout = 30000L)
- public void testPut() throws Exception {
- agent.configure(properties);
- agent.start();
- agent.put(EventBuilder.withBody(body, headers));
- Event event;
- while ((event = eventCollector.poll()) == null) {
- Thread.sleep(500L);
- }
- Assert.assertNotNull(event);
- Assert.assertArrayEquals(body, event.getBody());
- Assert.assertEquals(headers, event.getHeaders());
- }
-
- @Test(timeout = 30000L)
- public void testPutAll() throws Exception {
- List<Event> events = Lists.newArrayList();
- events.add(EventBuilder.withBody(body, headers));
- agent.configure(properties);
- agent.start();
- agent.putAll(events);
-
- Event event;
- while ((event = eventCollector.poll()) == null) {
- Thread.sleep(500L);
- }
- Assert.assertNotNull(event);
- Assert.assertArrayEquals(body, event.getBody());
- Assert.assertEquals(headers, event.getHeaders());
- }
-
- @Test(timeout = 30000L)
- public void testPutWithInterceptors() throws Exception {
- properties.put("source.interceptors", "i1");
- properties.put("source.interceptors.i1.type", "static");
- properties.put("source.interceptors.i1.key", "key2");
- properties.put("source.interceptors.i1.value", "value2");
-
- agent.configure(properties);
- agent.start();
- agent.put(EventBuilder.withBody(body, headers));
-
- Event event;
- while ((event = eventCollector.poll()) == null) {
- Thread.sleep(500L);
+ @After
+ public void tearDown() throws Exception {
+ if (agent != null) {
+ try {
+ agent.stop();
+ } catch (Exception e) {
+ LOGGER.debug("Error shutting down agent", e);
+ }
+ }
+ if (nettyServer != null) {
+ try {
+ nettyServer.close();
+ } catch (Exception e) {
+ LOGGER.debug("Error shutting down server", e);
+ }
+ }
}
- Assert.assertNotNull(event);
- Assert.assertArrayEquals(body, event.getBody());
- Map<String, String> newHeaders = new HashMap<String, String>(headers);
- newHeaders.put("key2", "value2");
- Assert.assertEquals(newHeaders, event.getHeaders());
- }
-
-
- @Test(timeout = 30000L)
- public void testEmbeddedAgentName() throws Exception {
- EmbeddedAgent embedAgent = new EmbeddedAgent("test 1 2" + serialNumber.incrementAndGet());
- List<Event> events = Lists.newArrayList();
- events.add(EventBuilder.withBody(body, headers));
- embedAgent.configure(properties);
- embedAgent.start();
- embedAgent.putAll(events);
-
- Event event;
- while ((event = eventCollector.poll()) == null) {
- Thread.sleep(500L);
+
+ @Test(timeout = 30000L)
+ public void testPut() throws Exception {
+ agent.configure(properties);
+ agent.start();
+ agent.put(EventBuilder.withBody(body, headers));
+
+ Event event;
+ while ((event = eventCollector.poll()) == null) {
+ Thread.sleep(500L);
+ }
+ Assert.assertNotNull(event);
+ Assert.assertArrayEquals(body, event.getBody());
+ Assert.assertEquals(headers, event.getHeaders());
}
- Assert.assertNotNull(event);
- Assert.assertArrayEquals(body, event.getBody());
- Assert.assertEquals(headers, event.getHeaders());
- if (embedAgent != null) {
- try {
- embedAgent.stop();
- } catch (Exception e) {
- LOGGER.debug("Error shutting down agent", e);
- }
+
+ @Test(timeout = 30000L)
+ public void testPutAll() throws Exception {
+ List<Event> events = Lists.newArrayList();
+ events.add(EventBuilder.withBody(body, headers));
+ agent.configure(properties);
+ agent.start();
+ agent.putAll(events);
+
+ Event event;
+ while ((event = eventCollector.poll()) == null) {
+ Thread.sleep(500L);
+ }
+ Assert.assertNotNull(event);
+ Assert.assertArrayEquals(body, event.getBody());
+ Assert.assertEquals(headers, event.getHeaders());
}
- }
-
- static class EventCollector implements AvroSourceProtocol {
- private final Queue<AvroFlumeEvent> eventQueue =
- new LinkedBlockingQueue<AvroFlumeEvent>();
-
- public Event poll() {
- AvroFlumeEvent avroEvent = eventQueue.poll();
- if (avroEvent != null) {
- return EventBuilder.withBody(avroEvent.getBody().array(),
- toStringMap(avroEvent.getHeaders()));
- }
- return null;
+
+ @Test(timeout = 30000L)
+ public void testPutWithInterceptors() throws Exception {
+ properties.put("source.interceptors", "i1");
+ properties.put("source.interceptors.i1.type", "static");
+ properties.put("source.interceptors.i1.key", "key2");
+ properties.put("source.interceptors.i1.value", "value2");
+
+ agent.configure(properties);
+ agent.start();
+ agent.put(EventBuilder.withBody(body, headers));
+
+ Event event;
+ while ((event = eventCollector.poll()) == null) {
+ Thread.sleep(500L);
+ }
+ Assert.assertNotNull(event);
+ Assert.assertArrayEquals(body, event.getBody());
+ Map<String, String> newHeaders = new HashMap<String, String>(headers);
+ newHeaders.put("key2", "value2");
+ Assert.assertEquals(newHeaders, event.getHeaders());
}
- @Override
- public Status append(AvroFlumeEvent event) {
- eventQueue.add(event);
- return Status.OK;
+
+ @Test(timeout = 30000L)
+ public void testEmbeddedAgentName() throws Exception {
+ EmbeddedAgent embedAgent = new EmbeddedAgent("test 1 2" + serialNumber.incrementAndGet());
+ List<Event> events = Lists.newArrayList();
+ events.add(EventBuilder.withBody(body, headers));
+ embedAgent.configure(properties);
+ embedAgent.start();
+ embedAgent.putAll(events);
+
+ Event event;
+ while ((event = eventCollector.poll()) == null) {
+ Thread.sleep(500L);
+ }
+ Assert.assertNotNull(event);
+ Assert.assertArrayEquals(body, event.getBody());
+ Assert.assertEquals(headers, event.getHeaders());
+ if (embedAgent != null) {
+ try {
+ embedAgent.stop();
+ } catch (Exception e) {
+ LOGGER.debug("Error shutting down agent", e);
+ }
+ }
}
- @Override
- public Status appendBatch(List<AvroFlumeEvent> events) {
- Preconditions.checkState(eventQueue.addAll(events));
- return Status.OK;
+
+ static class EventCollector implements AvroSourceProtocol {
+ private final Queue<AvroFlumeEvent> eventQueue = new LinkedBlockingQueue<AvroFlumeEvent>();
+
+ public Event poll() {
+ AvroFlumeEvent avroEvent = eventQueue.poll();
+ if (avroEvent != null) {
+ return EventBuilder.withBody(avroEvent.getBody().array(), toStringMap(avroEvent.getHeaders()));
+ }
+ return null;
+ }
+
+ @Override
+ public Status append(AvroFlumeEvent event) {
+ eventQueue.add(event);
+ return Status.OK;
+ }
+
+ @Override
+ public Status appendBatch(List<AvroFlumeEvent> events) {
+ Preconditions.checkState(eventQueue.addAll(events));
+ return Status.OK;
+ }
}
- }
- private static Map<String, String> toStringMap(Map<CharSequence, CharSequence> charSeqMap) {
- Map<String, String> stringMap = new HashMap<String, String>();
- for (Map.Entry<CharSequence, CharSequence> entry : charSeqMap.entrySet()) {
- stringMap.put(entry.getKey().toString(), entry.getValue().toString());
+ private static Map<String, String> toStringMap(Map<CharSequence, CharSequence> charSeqMap) {
+ Map<String, String> stringMap = new HashMap<String, String>();
+ for (Map.Entry<CharSequence, CharSequence> entry : charSeqMap.entrySet()) {
+ stringMap.put(entry.getKey().toString(), entry.getValue().toString());
+ }
+ return stringMap;
}
- return stringMap;
- }
- private static int findFreePort() throws IOException {
- try (ServerSocket socket = new ServerSocket(0)) {
- return socket.getLocalPort();
+ private static int findFreePort() throws IOException {
+ try (ServerSocket socket = new ServerSocket(0)) {
+ return socket.getLocalPort();
+ }
}
- }
}
diff --git a/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgentConfiguration.java b/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgentConfiguration.java
index ed26294..5a582b7 100644
--- a/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgentConfiguration.java
+++ b/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgentConfiguration.java
@@ -1,151 +1,154 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.agent.embedded;
+import com.google.common.collect.Maps;
import java.util.Map;
-
import junit.framework.Assert;
-
import org.apache.flume.FlumeException;
import org.junit.Before;
import org.junit.Test;
-import com.google.common.collect.Maps;
-
public class TestEmbeddedAgentConfiguration {
- private Map<String, String> properties;
-
- @Before
- public void setUp() throws Exception {
- properties = Maps.newHashMap();
- properties.put("source.type", EmbeddedAgentConfiguration.SOURCE_TYPE_EMBEDDED);
- properties.put("channel.type", "memory");
- properties.put("channel.capacity", "200");
- properties.put("sinks", "sink1 sink2");
- properties.put("sink1.type", "avro");
- properties.put("sink2.type", "avro");
- properties.put("sink1.hostname", "sink1.host");
- properties.put("sink1.port", "2");
- properties.put("sink2.hostname", "sink2.host");
- properties.put("sink2.port", "2");
- properties.put("processor.type", "load_balance");
- properties.put("source.interceptors", "i1");
- properties.put("source.interceptors.i1.type", "timestamp");
- }
-
- @Test
- public void testFullSourceType() throws Exception {
- doTestExcepted(EmbeddedAgentConfiguration.configure("test1", properties));
- }
-
- @Test
- public void testMissingSourceType() throws Exception {
- Assert.assertNotNull(properties.remove("source.type"));
- doTestExcepted(EmbeddedAgentConfiguration.configure("test1", properties));
- }
-
- @Test
- public void testShortSourceType() throws Exception {
- properties.put("source.type", "EMBEDDED");
- doTestExcepted(EmbeddedAgentConfiguration.configure("test1", properties));
- }
-
- public void doTestExcepted(Map<String, String> actual) throws Exception {
- Map<String, String> expected = Maps.newHashMap();
- expected.put("test1.channels", "channel-test1");
- expected.put("test1.channels.channel-test1.capacity", "200");
- expected.put("test1.channels.channel-test1.type", "memory");
- expected.put("test1.sinkgroups", "sink-group-test1");
- expected.put("test1.sinkgroups.sink-group-test1.processor.type", "load_balance");
- expected.put("test1.sinkgroups.sink-group-test1.sinks", "sink1 sink2");
- expected.put("test1.sinks", "sink1 sink2");
- expected.put("test1.sinks.sink1.channel", "channel-test1");
- expected.put("test1.sinks.sink1.hostname", "sink1.host");
- expected.put("test1.sinks.sink1.port", "2");
- expected.put("test1.sinks.sink1.type", "avro");
- expected.put("test1.sinks.sink2.channel", "channel-test1");
- expected.put("test1.sinks.sink2.hostname", "sink2.host");
- expected.put("test1.sinks.sink2.port", "2");
- expected.put("test1.sinks.sink2.type", "avro");
- expected.put("test1.sources", "source-test1");
- expected.put("test1.sources.source-test1.channels", "channel-test1");
- expected.put("test1.sources.source-test1.type",
- EmbeddedAgentConfiguration.SOURCE_TYPE_EMBEDDED);
- expected.put("test1.sources.source-test1.interceptors", "i1");
- expected.put("test1.sources.source-test1.interceptors.i1.type", "timestamp");
- Assert.assertEquals(expected, actual);
- }
-
- @Test(expected = FlumeException.class)
- public void testBadSource() throws Exception {
- properties.put("source.type", "exec");
- EmbeddedAgentConfiguration.configure("test1", properties);
- }
- @Test(expected = FlumeException.class)
- public void testBadChannel() throws Exception {
- properties.put("channel.type", "jdbc");
- EmbeddedAgentConfiguration.configure("test1", properties);
- }
- @Test(expected = FlumeException.class)
- public void testBadSink() throws Exception {
- properties.put("sink1.type", "hbase");
- EmbeddedAgentConfiguration.configure("test1", properties);
- }
- @Test(expected = FlumeException.class)
- public void testBadSinkProcessor() throws Exception {
- properties.put("processor.type", "bad");
- EmbeddedAgentConfiguration.configure("test1", properties);
- }
-
- @Test(expected = FlumeException.class)
- public void testNoChannel() throws Exception {
- properties.remove("channel.type");
- EmbeddedAgentConfiguration.configure("test1", properties);
- }
- @Test(expected = FlumeException.class)
- public void testNoSink() throws Exception {
- properties.remove("sink2.type");
- EmbeddedAgentConfiguration.configure("test1", properties);
- }
- @Test(expected = FlumeException.class)
- public void testNoSinkProcessor() throws Exception {
- properties.remove("processor.type");
- EmbeddedAgentConfiguration.configure("test1", properties);
- }
- @Test(expected = FlumeException.class)
- public void testBadKey() throws Exception {
- properties.put("bad.key.name", "bad");
- EmbeddedAgentConfiguration.configure("test1", properties);
- }
- @Test(expected = FlumeException.class)
- public void testSinkNamedLikeSource() throws Exception {
- properties.put("sinks", "source");
- EmbeddedAgentConfiguration.configure("test1", properties);
- }
- @Test(expected = FlumeException.class)
- public void testSinkNamedLikeChannel() throws Exception {
- properties.put("sinks", "channel");
- EmbeddedAgentConfiguration.configure("test1", properties);
- }
- @Test(expected = FlumeException.class)
- public void testSinkNamedLikeProcessor() throws Exception {
- properties.put("sinks", "processor");
- EmbeddedAgentConfiguration.configure("test1", properties);
- }
-}
\ No newline at end of file
+ private Map<String, String> properties;
+
+ @Before
+ public void setUp() throws Exception {
+ properties = Maps.newHashMap();
+ properties.put("source.type", EmbeddedAgentConfiguration.SOURCE_TYPE_EMBEDDED);
+ properties.put("channel.type", "memory");
+ properties.put("channel.capacity", "200");
+ properties.put("sinks", "sink1 sink2");
+ properties.put("sink1.type", "avro");
+ properties.put("sink2.type", "avro");
+ properties.put("sink1.hostname", "sink1.host");
+ properties.put("sink1.port", "2");
+ properties.put("sink2.hostname", "sink2.host");
+ properties.put("sink2.port", "2");
+ properties.put("processor.type", "load_balance");
+ properties.put("source.interceptors", "i1");
+ properties.put("source.interceptors.i1.type", "timestamp");
+ }
+
+ @Test
+ public void testFullSourceType() throws Exception {
+ doTestExcepted(EmbeddedAgentConfiguration.configure("test1", properties));
+ }
+
+ @Test
+ public void testMissingSourceType() throws Exception {
+ Assert.assertNotNull(properties.remove("source.type"));
+ doTestExcepted(EmbeddedAgentConfiguration.configure("test1", properties));
+ }
+
+ @Test
+ public void testShortSourceType() throws Exception {
+ properties.put("source.type", "EMBEDDED");
+ doTestExcepted(EmbeddedAgentConfiguration.configure("test1", properties));
+ }
+
+ public void doTestExcepted(Map<String, String> actual) throws Exception {
+ Map<String, String> expected = Maps.newHashMap();
+ expected.put("test1.channels", "channel-test1");
+ expected.put("test1.channels.channel-test1.capacity", "200");
+ expected.put("test1.channels.channel-test1.type", "memory");
+ expected.put("test1.sinkgroups", "sink-group-test1");
+ expected.put("test1.sinkgroups.sink-group-test1.processor.type", "load_balance");
+ expected.put("test1.sinkgroups.sink-group-test1.sinks", "sink1 sink2");
+ expected.put("test1.sinks", "sink1 sink2");
+ expected.put("test1.sinks.sink1.channel", "channel-test1");
+ expected.put("test1.sinks.sink1.hostname", "sink1.host");
+ expected.put("test1.sinks.sink1.port", "2");
+ expected.put("test1.sinks.sink1.type", "avro");
+ expected.put("test1.sinks.sink2.channel", "channel-test1");
+ expected.put("test1.sinks.sink2.hostname", "sink2.host");
+ expected.put("test1.sinks.sink2.port", "2");
+ expected.put("test1.sinks.sink2.type", "avro");
+ expected.put("test1.sources", "source-test1");
+ expected.put("test1.sources.source-test1.channels", "channel-test1");
+ expected.put("test1.sources.source-test1.type", EmbeddedAgentConfiguration.SOURCE_TYPE_EMBEDDED);
+ expected.put("test1.sources.source-test1.interceptors", "i1");
+ expected.put("test1.sources.source-test1.interceptors.i1.type", "timestamp");
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test(expected = FlumeException.class)
+ public void testBadSource() throws Exception {
+ properties.put("source.type", "exec");
+ EmbeddedAgentConfiguration.configure("test1", properties);
+ }
+
+ @Test(expected = FlumeException.class)
+ public void testBadChannel() throws Exception {
+ properties.put("channel.type", "jdbc");
+ EmbeddedAgentConfiguration.configure("test1", properties);
+ }
+
+ @Test(expected = FlumeException.class)
+ public void testBadSink() throws Exception {
+ properties.put("sink1.type", "hbase");
+ EmbeddedAgentConfiguration.configure("test1", properties);
+ }
+
+ @Test(expected = FlumeException.class)
+ public void testBadSinkProcessor() throws Exception {
+ properties.put("processor.type", "bad");
+ EmbeddedAgentConfiguration.configure("test1", properties);
+ }
+
+ @Test(expected = FlumeException.class)
+ public void testNoChannel() throws Exception {
+ properties.remove("channel.type");
+ EmbeddedAgentConfiguration.configure("test1", properties);
+ }
+
+ @Test(expected = FlumeException.class)
+ public void testNoSink() throws Exception {
+ properties.remove("sink2.type");
+ EmbeddedAgentConfiguration.configure("test1", properties);
+ }
+
+ @Test(expected = FlumeException.class)
+ public void testNoSinkProcessor() throws Exception {
+ properties.remove("processor.type");
+ EmbeddedAgentConfiguration.configure("test1", properties);
+ }
+
+ @Test(expected = FlumeException.class)
+ public void testBadKey() throws Exception {
+ properties.put("bad.key.name", "bad");
+ EmbeddedAgentConfiguration.configure("test1", properties);
+ }
+
+ @Test(expected = FlumeException.class)
+ public void testSinkNamedLikeSource() throws Exception {
+ properties.put("sinks", "source");
+ EmbeddedAgentConfiguration.configure("test1", properties);
+ }
+
+ @Test(expected = FlumeException.class)
+ public void testSinkNamedLikeChannel() throws Exception {
+ properties.put("sinks", "channel");
+ EmbeddedAgentConfiguration.configure("test1", properties);
+ }
+
+ @Test(expected = FlumeException.class)
+ public void testSinkNamedLikeProcessor() throws Exception {
+ properties.put("sinks", "processor");
+ EmbeddedAgentConfiguration.configure("test1", properties);
+ }
+}
diff --git a/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgentEmbeddedSource.java b/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgentEmbeddedSource.java
index ae59444..65b905e 100644
--- a/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgentEmbeddedSource.java
+++ b/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgentEmbeddedSource.java
@@ -1,30 +1,29 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.agent.embedded;
import static org.mockito.Mockito.*;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import java.util.List;
import java.util.Map;
-
import junit.framework.Assert;
-
import org.apache.flume.Channel;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
@@ -36,174 +35,172 @@ import org.apache.flume.node.MaterializedConfiguration;
import org.junit.Before;
import org.junit.Test;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
public class TestEmbeddedAgentEmbeddedSource {
- private EmbeddedAgent agent;
- private Map<String, String> properties;
-
- private MaterializedConfiguration config;
- private EmbeddedSource source;
- private SourceRunner sourceRunner;
- private Channel channel;
- private SinkRunner sinkRunner;
-
- @Before
- public void setUp() throws Exception {
-
- properties = Maps.newHashMap();
- properties.put("source.type", EmbeddedAgentConfiguration.SOURCE_TYPE_EMBEDDED);
- properties.put("channel.type", "memory");
- properties.put("sinks", "sink1 sink2");
- properties.put("sink1.type", "avro");
- properties.put("sink2.type", "avro");
- properties.put("processor.type", "load_balance");
-
- sourceRunner = mock(SourceRunner.class);
- channel = mock(Channel.class);
- sinkRunner = mock(SinkRunner.class);
-
- source = mock(EmbeddedSource.class);
- when(sourceRunner.getSource()).thenReturn(source);
-
- when(sourceRunner.getLifecycleState()).thenReturn(LifecycleState.START);
- when(channel.getLifecycleState()).thenReturn(LifecycleState.START);
- when(sinkRunner.getLifecycleState()).thenReturn(LifecycleState.START);
-
- config = new MaterializedConfiguration() {
- @Override
- public Map<String, SourceRunner> getSourceRunners() {
- Map<String, SourceRunner> result = Maps.newHashMap();
- result.put("source", sourceRunner);
- return ImmutableMap.copyOf(result);
- }
-
- @Override
- public Map<String, SinkRunner> getSinkRunners() {
- Map<String, SinkRunner> result = Maps.newHashMap();
- result.put("sink", sinkRunner);
- return ImmutableMap.copyOf(result);
- }
-
- @Override
- public Map<String, Channel> getChannels() {
- Map<String, Channel> result = Maps.newHashMap();
- result.put("channel", channel);
- return ImmutableMap.copyOf(result);
- }
-
- @Override
- public void addSourceRunner(String name, SourceRunner sourceRunner) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void addSinkRunner(String name, SinkRunner sinkRunner) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void addChannel(String name, Channel channel) {
- throw new UnsupportedOperationException();
- }
- };
- agent = new EmbeddedAgent(new MaterializedConfigurationProvider() {
- public MaterializedConfiguration get(String name, Map<String, String> properties) {
- return config;
- }
- }, "dummy");
- }
-
- @Test
- public void testStart() {
- agent.configure(properties);
- agent.start();
- verify(sourceRunner, times(1)).start();
- verify(channel, times(1)).start();
- verify(sinkRunner, times(1)).start();
- }
-
- @Test
- public void testStop() {
- agent.configure(properties);
- agent.start();
- agent.stop();
- verify(sourceRunner, times(1)).stop();
- verify(channel, times(1)).stop();
- verify(sinkRunner, times(1)).stop();
- }
-
- @Test
- public void testStartSourceThrowsException() {
- doThrow(new LocalRuntimeException()).when(sourceRunner).start();
- startExpectingLocalRuntimeException();
- }
-
- @Test
- public void testStartChannelThrowsException() {
- doThrow(new LocalRuntimeException()).when(channel).start();
- startExpectingLocalRuntimeException();
- }
-
- @Test
- public void testStartSinkThrowsException() {
- doThrow(new LocalRuntimeException()).when(sinkRunner).start();
- startExpectingLocalRuntimeException();
- }
-
- private void startExpectingLocalRuntimeException() {
- agent.configure(properties);
- try {
- agent.start();
- Assert.fail();
- } catch (LocalRuntimeException e) {
- // expected
+ private EmbeddedAgent agent;
+ private Map<String, String> properties;
+
+ private MaterializedConfiguration config;
+ private EmbeddedSource source;
+ private SourceRunner sourceRunner;
+ private Channel channel;
+ private SinkRunner sinkRunner;
+
+ @Before
+ public void setUp() throws Exception {
+
+ properties = Maps.newHashMap();
+ properties.put("source.type", EmbeddedAgentConfiguration.SOURCE_TYPE_EMBEDDED);
+ properties.put("channel.type", "memory");
+ properties.put("sinks", "sink1 sink2");
+ properties.put("sink1.type", "avro");
+ properties.put("sink2.type", "avro");
+ properties.put("processor.type", "load_balance");
+
+ sourceRunner = mock(SourceRunner.class);
+ channel = mock(Channel.class);
+ sinkRunner = mock(SinkRunner.class);
+
+ source = mock(EmbeddedSource.class);
+ when(sourceRunner.getSource()).thenReturn(source);
+
+ when(sourceRunner.getLifecycleState()).thenReturn(LifecycleState.START);
+ when(channel.getLifecycleState()).thenReturn(LifecycleState.START);
+ when(sinkRunner.getLifecycleState()).thenReturn(LifecycleState.START);
+
+ config = new MaterializedConfiguration() {
+ @Override
+ public Map<String, SourceRunner> getSourceRunners() {
+ Map<String, SourceRunner> result = Maps.newHashMap();
+ result.put("source", sourceRunner);
+ return ImmutableMap.copyOf(result);
+ }
+
+ @Override
+ public Map<String, SinkRunner> getSinkRunners() {
+ Map<String, SinkRunner> result = Maps.newHashMap();
+ result.put("sink", sinkRunner);
+ return ImmutableMap.copyOf(result);
+ }
+
+ @Override
+ public Map<String, Channel> getChannels() {
+ Map<String, Channel> result = Maps.newHashMap();
+ result.put("channel", channel);
+ return ImmutableMap.copyOf(result);
+ }
+
+ @Override
+ public void addSourceRunner(String name, SourceRunner sourceRunner) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void addSinkRunner(String name, SinkRunner sinkRunner) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void addChannel(String name, Channel channel) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ agent = new EmbeddedAgent(
+ new MaterializedConfigurationProvider() {
+ public MaterializedConfiguration get(String name, Map<String, String> properties) {
+ return config;
+ }
+ },
+ "dummy");
+ }
+
+ @Test
+ public void testStart() {
+ agent.configure(properties);
+ agent.start();
+ verify(sourceRunner, times(1)).start();
+ verify(channel, times(1)).start();
+ verify(sinkRunner, times(1)).start();
+ }
+
+ @Test
+ public void testStop() {
+ agent.configure(properties);
+ agent.start();
+ agent.stop();
+ verify(sourceRunner, times(1)).stop();
+ verify(channel, times(1)).stop();
+ verify(sinkRunner, times(1)).stop();
+ }
+
+ @Test
+ public void testStartSourceThrowsException() {
+ doThrow(new LocalRuntimeException()).when(sourceRunner).start();
+ startExpectingLocalRuntimeException();
+ }
+
+ @Test
+ public void testStartChannelThrowsException() {
+ doThrow(new LocalRuntimeException()).when(channel).start();
+ startExpectingLocalRuntimeException();
+ }
+
+ @Test
+ public void testStartSinkThrowsException() {
+ doThrow(new LocalRuntimeException()).when(sinkRunner).start();
+ startExpectingLocalRuntimeException();
+ }
+
+ private void startExpectingLocalRuntimeException() {
+ agent.configure(properties);
+ try {
+ agent.start();
+ Assert.fail();
+ } catch (LocalRuntimeException e) {
+ // expected
+ }
+ verify(sourceRunner, times(1)).stop();
+ verify(channel, times(1)).stop();
+ verify(sinkRunner, times(1)).stop();
+ }
+
+ private static class LocalRuntimeException extends RuntimeException {
+ private static final long serialVersionUID = 116546244849853151L;
+ }
+
+ @Test
+ public void testPut() throws EventDeliveryException {
+ Event event = new SimpleEvent();
+ agent.configure(properties);
+ agent.start();
+ agent.put(event);
+ verify(source, times(1)).put(event);
+ }
+
+ @Test
+ public void testPutAll() throws EventDeliveryException {
+ Event event = new SimpleEvent();
+ List<Event> events = Lists.newArrayList();
+ events.add(event);
+ agent.configure(properties);
+ agent.start();
+ agent.putAll(events);
+ verify(source, times(1)).putAll(events);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testPutNotStarted() throws EventDeliveryException {
+ Event event = new SimpleEvent();
+ agent.configure(properties);
+ agent.put(event);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testPutAllNotStarted() throws EventDeliveryException {
+ Event event = new SimpleEvent();
+ List<Event> events = Lists.newArrayList();
+ events.add(event);
+ agent.configure(properties);
+ agent.putAll(events);
}
- verify(sourceRunner, times(1)).stop();
- verify(channel, times(1)).stop();
- verify(sinkRunner, times(1)).stop();
- }
-
- private static class LocalRuntimeException extends RuntimeException {
- private static final long serialVersionUID = 116546244849853151L;
- }
-
- @Test
- public void testPut() throws EventDeliveryException {
- Event event = new SimpleEvent();
- agent.configure(properties);
- agent.start();
- agent.put(event);
- verify(source, times(1)).put(event);
- }
-
- @Test
- public void testPutAll() throws EventDeliveryException {
- Event event = new SimpleEvent();
- List<Event> events = Lists.newArrayList();
- events.add(event);
- agent.configure(properties);
- agent.start();
- agent.putAll(events);
- verify(source, times(1)).putAll(events);
- }
-
- @Test(expected = IllegalStateException.class)
- public void testPutNotStarted() throws EventDeliveryException {
- Event event = new SimpleEvent();
- agent.configure(properties);
- agent.put(event);
- }
-
- @Test(expected = IllegalStateException.class)
- public void testPutAllNotStarted() throws EventDeliveryException {
- Event event = new SimpleEvent();
- List<Event> events = Lists.newArrayList();
- events.add(event);
- agent.configure(properties);
- agent.putAll(events);
- }
}
diff --git a/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgentState.java b/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgentState.java
index 0f0ad23..63fbda4 100644
--- a/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgentState.java
+++ b/flume-embedded-agent/src/test/java/org/apache/flume/agent/embedded/TestEmbeddedAgentState.java
@@ -1,125 +1,122 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.agent.embedded;
import com.google.common.base.Throwables;
import com.google.common.collect.Maps;
+import java.util.Map;
import org.apache.flume.FlumeException;
import org.junit.Before;
import org.junit.Test;
-import java.util.Map;
-
public class TestEmbeddedAgentState {
- private static final String HOSTNAME = "localhost";
- private EmbeddedAgent agent;
- private Map<String, String> properties;
+ private static final String HOSTNAME = "localhost";
+ private EmbeddedAgent agent;
+ private Map<String, String> properties;
- @Before
- public void setUp() throws Exception {
- agent = new EmbeddedAgent("dummy");
- properties = Maps.newHashMap();
- properties.put("source.type", EmbeddedAgentConfiguration.SOURCE_TYPE_EMBEDDED);
- properties.put("channel.type", "memory");
- properties.put("sinks", "sink1 sink2");
- properties.put("sink1.type", "avro");
- properties.put("sink2.type", "avro");
- properties.put("sink1.hostname", HOSTNAME);
- properties.put("sink1.port", "0");
- properties.put("sink2.hostname", HOSTNAME);
- properties.put("sink2.port", "0");
- properties.put("processor.type", "load_balance");
- }
+ @Before
+ public void setUp() throws Exception {
+ agent = new EmbeddedAgent("dummy");
+ properties = Maps.newHashMap();
+ properties.put("source.type", EmbeddedAgentConfiguration.SOURCE_TYPE_EMBEDDED);
+ properties.put("channel.type", "memory");
+ properties.put("sinks", "sink1 sink2");
+ properties.put("sink1.type", "avro");
+ properties.put("sink2.type", "avro");
+ properties.put("sink1.hostname", HOSTNAME);
+ properties.put("sink1.port", "0");
+ properties.put("sink2.hostname", HOSTNAME);
+ properties.put("sink2.port", "0");
+ properties.put("processor.type", "load_balance");
+ }
- @Test(expected = FlumeException.class)
- public void testConfigureWithBadSourceType() {
- properties.put(EmbeddedAgentConfiguration.SOURCE_TYPE, "bad");
- agent.configure(properties);
- }
+ @Test(expected = FlumeException.class)
+ public void testConfigureWithBadSourceType() {
+ properties.put(EmbeddedAgentConfiguration.SOURCE_TYPE, "bad");
+ agent.configure(properties);
+ }
- @Test(expected = IllegalStateException.class)
- public void testConfigureWhileStarted() {
- try {
- agent.configure(properties);
- agent.start();
- } catch (Exception e) {
- Throwables.propagate(e);
+ @Test(expected = IllegalStateException.class)
+ public void testConfigureWhileStarted() {
+ try {
+ agent.configure(properties);
+ agent.start();
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ }
+ agent.configure(properties);
}
- agent.configure(properties);
- }
- @Test
- public void testConfigureMultipleTimes() {
- agent.configure(properties);
- agent.configure(properties);
- }
+ @Test
+ public void testConfigureMultipleTimes() {
+ agent.configure(properties);
+ agent.configure(properties);
+ }
- @Test(expected = IllegalStateException.class)
- public void testStartWhileStarted() {
- try {
- agent.configure(properties);
- agent.start();
- } catch (Exception e) {
- Throwables.propagate(e);
+ @Test(expected = IllegalStateException.class)
+ public void testStartWhileStarted() {
+ try {
+ agent.configure(properties);
+ agent.start();
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ }
+ agent.start();
}
- agent.start();
- }
- @Test(expected = IllegalStateException.class)
- public void testStartUnconfigured() {
- agent.start();
- }
+ @Test(expected = IllegalStateException.class)
+ public void testStartUnconfigured() {
+ agent.start();
+ }
- @Test(expected = IllegalStateException.class)
- public void testStopBeforeConfigure() {
- agent.stop();
- }
+ @Test(expected = IllegalStateException.class)
+ public void testStopBeforeConfigure() {
+ agent.stop();
+ }
- @Test(expected = IllegalStateException.class)
- public void testStoppedWhileStopped() {
- try {
- agent.configure(properties);
- } catch (Exception e) {
- Throwables.propagate(e);
+ @Test(expected = IllegalStateException.class)
+ public void testStoppedWhileStopped() {
+ try {
+ agent.configure(properties);
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ }
+ agent.stop();
}
- agent.stop();
- }
- @Test(expected = IllegalStateException.class)
- public void testStopAfterStop() {
- try {
- agent.configure(properties);
- agent.start();
- agent.stop();
- } catch (Exception e) {
- Throwables.propagate(e);
+ @Test(expected = IllegalStateException.class)
+ public void testStopAfterStop() {
+ try {
+ agent.configure(properties);
+ agent.start();
+ agent.stop();
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ }
+ agent.stop();
}
- agent.stop();
- }
- @Test(expected = IllegalStateException.class)
- public void testStopAfterConfigure() {
- try {
- agent.configure(properties);
- } catch (Exception e) {
- Throwables.propagate(e);
+ @Test(expected = IllegalStateException.class)
+ public void testStopAfterConfigure() {
+ try {
+ agent.configure(properties);
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ }
+ agent.stop();
}
- agent.stop();
- }
}
diff --git a/flume-embedded-agent/src/test/resources/log4j2.xml b/flume-embedded-agent/src/test/resources/log4j2.xml
index fbc44b4..ac0d619 100644
--- a/flume-embedded-agent/src/test/resources/log4j2.xml
+++ b/flume-embedded-agent/src/test/resources/log4j2.xml
@@ -1,21 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to you under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
<Configuration status="OFF">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
@@ -28,4 +27,4 @@
<AppenderRef ref="Console" />
</Root>
</Loggers>
-</Configuration>
\ No newline at end of file
+</Configuration>
diff --git a/flume-rpc-avro/pom.xml b/flume-rpc-avro/pom.xml
index 8c672a4..769e4a5 100644
--- a/flume-rpc-avro/pom.xml
+++ b/flume-rpc-avro/pom.xml
@@ -1,23 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to you under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -27,7 +25,6 @@ limitations under the License.
<relativePath>../pom.xml</relativePath>
</parent>
- <groupId>org.apache.flume</groupId>
<artifactId>flume-rpc-avro</artifactId>
<name>Flume RPC Avro</name>
@@ -35,96 +32,98 @@ limitations under the License.
<module.name>org.apache.flume.rpc.avro</module.name>
<spotbugs.maxAllowedViolations>55</spotbugs.maxAllowedViolations>
<pmd.maxAllowedViolations>121</pmd.maxAllowedViolations>
+
+ <!-- Temporary property -->
</properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-ipc-netty</artifactId>
- </dependency>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-ipc-netty</artifactId>
+ </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-sdk</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-sdk</artifactId>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-configuration</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-node</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>${dropwizard-metrics.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-sdk</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-sdk</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-configuration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-node</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${dropwizard-metrics.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <scope>test</scope>
+ </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <scope>test</scope>
+ </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-jcl</artifactId>
- </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-jcl</artifactId>
+ </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>${junit.version}</version>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
<build>
<plugins>
<plugin>
@@ -176,10 +175,10 @@ limitations under the License.
<executions>
<execution>
<id>verify.rat</id>
- <phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
+ <phase>verify</phase>
</execution>
</executions>
</plugin>
@@ -192,11 +191,11 @@ limitations under the License.
<executions>
<execution>
<id>Avro Generate</id>
- <phase>generate-sources</phase>
<goals>
<goal>idl-protocol</goal>
<goal>schema</goal>
</goals>
+ <phase>generate-sources</phase>
<configuration>
<sourceDirectory>${project.basedir}/src/main/avro</sourceDirectory>
<outputDirectory>${project.build.directory}/generated-sources/avro</outputDirectory>
diff --git a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/client/AvroCLIClient.java b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/client/AvroCLIClient.java
index 35288f7..5bfef01 100644
--- a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/client/AvroCLIClient.java
+++ b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/client/AvroCLIClient.java
@@ -1,24 +1,22 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
-
package org.apache.flume.rpc.avro.client;
+import com.google.common.base.Preconditions;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
@@ -28,8 +26,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-
-import com.google.common.base.Preconditions;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
@@ -56,199 +52,196 @@ import org.slf4j.LoggerFactory;
@InterfaceStability.Evolving
public class AvroCLIClient {
- private static final Logger logger = LoggerFactory
- .getLogger(AvroCLIClient.class);
-
- private static final int BATCH_SIZE = 5;
- private static final int MAX_LINE_LENGTH = 2000;
-
- private String hostname;
- private int port;
- private String fileName;
- private String rpcClientPropsFile;
- private String dirName;
- private Map<String, String> headers = new HashMap<String, String>();
- private int sent;
-
- public static void main(String[] args) {
- SSLUtil.initGlobalSSLParameters();
-
- AvroCLIClient client = new AvroCLIClient();
-
- try {
- if (client.parseCommandLine(args)) {
- client.run();
- }
- } catch (ParseException e) {
- logger.error("Unable to parse command line options - {}", e.getMessage());
- } catch (IOException e) {
- logger.error("Unable to send data to Flume. Exception follows.", e);
- } catch (FlumeException e) {
- logger.error("Unable to open connection to Flume. Exception follows.", e);
- } catch (EventDeliveryException e) {
- logger.error("Unable to deliver events to Flume. Exception follows.", e);
- }
+ private static final Logger logger = LoggerFactory.getLogger(AvroCLIClient.class);
+
+ private static final int BATCH_SIZE = 5;
+ private static final int MAX_LINE_LENGTH = 2000;
+
+ private String hostname;
+ private int port;
+ private String fileName;
+ private String rpcClientPropsFile;
+ private String dirName;
+ private Map<String, String> headers = new HashMap<String, String>();
+ private int sent;
+
+ public static void main(String[] args) {
+ SSLUtil.initGlobalSSLParameters();
+
+ AvroCLIClient client = new AvroCLIClient();
- logger.debug("Exiting");
- }
-
- /*
- * Header Format : key1=value1, key2=value2,...
- */
- private void parseHeaders(CommandLine commandLine) {
- String headerFile = commandLine.getOptionValue("headerFile");
- FileInputStream fs = null;
- try {
- if (headerFile != null) {
- fs = new FileInputStream(headerFile);
- Properties properties = new Properties();
- properties.load(fs);
- for (Map.Entry<Object, Object> propertiesEntry : properties.entrySet()) {
- String key = (String) propertiesEntry.getKey();
- String value = (String) propertiesEntry.getValue();
- logger.debug("Inserting Header Key [" + key + "] header value [" + value + "]");
- headers.put(key, value);
- }
- }
- } catch (Exception e) {
- logger.error("Unable to load headerFile", headerFile, e);
- return;
- } finally {
- if (fs != null) {
try {
- fs.close();
- } catch (Exception e) {
- logger.error("Unable to close headerFile", e);
- return;
+ if (client.parseCommandLine(args)) {
+ client.run();
+ }
+ } catch (ParseException e) {
+ logger.error("Unable to parse command line options - {}", e.getMessage());
+ } catch (IOException e) {
+ logger.error("Unable to send data to Flume. Exception follows.", e);
+ } catch (FlumeException e) {
+ logger.error("Unable to open connection to Flume. Exception follows.", e);
+ } catch (EventDeliveryException e) {
+ logger.error("Unable to deliver events to Flume. Exception follows.", e);
}
- }
- }
- }
-
- private boolean parseCommandLine(String[] args) throws ParseException {
- Options options = new Options();
-
- options
- .addOption("P", "rpcProps", true, "RPC client properties file with " +
- "server connection params")
- .addOption("p", "port", true, "port of the avro source")
- .addOption("H", "host", true, "hostname of the avro source")
- .addOption("F", "filename", true, "file to stream to avro source")
- .addOption(null, "dirname", true, "directory to stream to avro source")
- .addOption("R", "headerFile", true, ("file containing headers as " +
- "key/value pairs on each new line"))
- .addOption("h", "help", false, "display help text");
-
- CommandLineParser parser = new GnuParser();
- CommandLine commandLine = parser.parse(options, args);
-
- if (commandLine.hasOption('h')) {
- new HelpFormatter().printHelp("flume-ng avro-client", "", options,
- "The --dirname option assumes that a spooling directory exists " +
- "where immutable log files are dropped.", true);
-
- return false;
- }
- if (commandLine.hasOption("filename") && commandLine.hasOption("dirname")) {
- throw new ParseException(
- "--filename and --dirname options cannot be used simultaneously");
+ logger.debug("Exiting");
}
- if (!commandLine.hasOption("port") && !commandLine.hasOption("host") &&
- !commandLine.hasOption("rpcProps")) {
- throw new ParseException("Either --rpcProps or both --host and --port " +
- "must be specified.");
+ /*
+ * Header Format : key1=value1, key2=value2,...
+ */
+ private void parseHeaders(CommandLine commandLine) {
+ String headerFile = commandLine.getOptionValue("headerFile");
+ FileInputStream fs = null;
+ try {
+ if (headerFile != null) {
+ fs = new FileInputStream(headerFile);
+ Properties properties = new Properties();
+ properties.load(fs);
+ for (Map.Entry<Object, Object> propertiesEntry : properties.entrySet()) {
+ String key = (String) propertiesEntry.getKey();
+ String value = (String) propertiesEntry.getValue();
+ logger.debug("Inserting Header Key [" + key + "] header value [" + value + "]");
+ headers.put(key, value);
+ }
+ }
+ } catch (Exception e) {
+ logger.error("Unable to load headerFile", headerFile, e);
+ return;
+ } finally {
+ if (fs != null) {
+ try {
+ fs.close();
+ } catch (Exception e) {
+ logger.error("Unable to close headerFile", e);
+ return;
+ }
+ }
+ }
}
- if (commandLine.hasOption("rpcProps")) {
- rpcClientPropsFile = commandLine.getOptionValue("rpcProps");
- Preconditions.checkNotNull(rpcClientPropsFile, "RPC client properties " +
- "file must be specified after --rpcProps argument.");
- Preconditions.checkArgument(new File(rpcClientPropsFile).exists(),
- "RPC client properties file %s does not exist!", rpcClientPropsFile);
- }
+ private boolean parseCommandLine(String[] args) throws ParseException {
+ Options options = new Options();
+
+ options.addOption("P", "rpcProps", true, "RPC client properties file with " + "server connection params")
+ .addOption("p", "port", true, "port of the avro source")
+ .addOption("H", "host", true, "hostname of the avro source")
+ .addOption("F", "filename", true, "file to stream to avro source")
+ .addOption(null, "dirname", true, "directory to stream to avro source")
+ .addOption(
+ "R", "headerFile", true, ("file containing headers as " + "key/value pairs on each new line"))
+ .addOption("h", "help", false, "display help text");
+
+ CommandLineParser parser = new GnuParser();
+ CommandLine commandLine = parser.parse(options, args);
+
+ if (commandLine.hasOption('h')) {
+ new HelpFormatter()
+ .printHelp(
+ "flume-ng avro-client",
+ "",
+ options,
+ "The --dirname option assumes that a spooling directory exists "
+ + "where immutable log files are dropped.",
+ true);
+
+ return false;
+ }
- if (rpcClientPropsFile == null) {
- if (!commandLine.hasOption("port")) {
- throw new ParseException(
- "You must specify a port to connect to with --port");
- }
- port = Integer.parseInt(commandLine.getOptionValue("port"));
-
- if (!commandLine.hasOption("host")) {
- throw new ParseException(
- "You must specify a hostname to connect to with --host");
- }
- hostname = commandLine.getOptionValue("host");
- }
+ if (commandLine.hasOption("filename") && commandLine.hasOption("dirname")) {
+ throw new ParseException("--filename and --dirname options cannot be used simultaneously");
+ }
- fileName = commandLine.getOptionValue("filename");
- dirName = commandLine.getOptionValue("dirname");
+ if (!commandLine.hasOption("port") && !commandLine.hasOption("host") && !commandLine.hasOption("rpcProps")) {
+ throw new ParseException("Either --rpcProps or both --host and --port " + "must be specified.");
+ }
- if (commandLine.hasOption("headerFile")) {
- parseHeaders(commandLine);
- }
+ if (commandLine.hasOption("rpcProps")) {
+ rpcClientPropsFile = commandLine.getOptionValue("rpcProps");
+ Preconditions.checkNotNull(
+ rpcClientPropsFile, "RPC client properties " + "file must be specified after --rpcProps argument.");
+ Preconditions.checkArgument(
+ new File(rpcClientPropsFile).exists(),
+ "RPC client properties file %s does not exist!",
+ rpcClientPropsFile);
+ }
- return true;
- }
+ if (rpcClientPropsFile == null) {
+ if (!commandLine.hasOption("port")) {
+ throw new ParseException("You must specify a port to connect to with --port");
+ }
+ port = Integer.parseInt(commandLine.getOptionValue("port"));
- private void run() throws IOException, FlumeException,
- EventDeliveryException {
+ if (!commandLine.hasOption("host")) {
+ throw new ParseException("You must specify a hostname to connect to with --host");
+ }
+ hostname = commandLine.getOptionValue("host");
+ }
+
+ fileName = commandLine.getOptionValue("filename");
+ dirName = commandLine.getOptionValue("dirname");
- EventReader reader = null;
+ if (commandLine.hasOption("headerFile")) {
+ parseHeaders(commandLine);
+ }
- RpcClient rpcClient;
- if (rpcClientPropsFile != null) {
- rpcClient = RpcClientFactory.getInstance(new File(rpcClientPropsFile));
- } else {
- rpcClient = RpcClientFactory.getDefaultInstance(hostname, port,
- BATCH_SIZE);
+ return true;
}
- try {
- if (fileName != null) {
- reader = new SimpleTextLineEventReader(new FileReader(new File(fileName)));
- } else if (dirName != null) {
- reader = new ReliableSpoolingFileEventReader.Builder()
- .spoolDirectory(new File(dirName))
- .sourceCounter(new SourceCounter("avrocli"))
- .build();
- } else {
- reader = new SimpleTextLineEventReader(new InputStreamReader(System.in));
- }
-
- long lastCheck = System.currentTimeMillis();
- long sentBytes = 0;
-
- int batchSize = rpcClient.getBatchSize();
- List<Event> events;
- while (!(events = reader.readEvents(batchSize)).isEmpty()) {
- for (Event event : events) {
- event.setHeaders(headers);
- sentBytes += event.getBody().length;
- sent++;
-
- long now = System.currentTimeMillis();
- if (now >= lastCheck + 5000) {
- logger.debug("Packed {} bytes, {} events", sentBytes, sent);
- lastCheck = now;
- }
+ private void run() throws IOException, FlumeException, EventDeliveryException {
+
+ EventReader reader = null;
+
+ RpcClient rpcClient;
+ if (rpcClientPropsFile != null) {
+ rpcClient = RpcClientFactory.getInstance(new File(rpcClientPropsFile));
+ } else {
+ rpcClient = RpcClientFactory.getDefaultInstance(hostname, port, BATCH_SIZE);
}
- rpcClient.appendBatch(events);
- if (reader instanceof ReliableEventReader) {
- ((ReliableEventReader) reader).commit();
+
+ try {
+ if (fileName != null) {
+ reader = new SimpleTextLineEventReader(new FileReader(new File(fileName)));
+ } else if (dirName != null) {
+ reader = new ReliableSpoolingFileEventReader.Builder()
+ .spoolDirectory(new File(dirName))
+ .sourceCounter(new SourceCounter("avrocli"))
+ .build();
+ } else {
+ reader = new SimpleTextLineEventReader(new InputStreamReader(System.in));
+ }
+
+ long lastCheck = System.currentTimeMillis();
+ long sentBytes = 0;
+
+ int batchSize = rpcClient.getBatchSize();
+ List<Event> events;
+ while (!(events = reader.readEvents(batchSize)).isEmpty()) {
+ for (Event event : events) {
+ event.setHeaders(headers);
+ sentBytes += event.getBody().length;
+ sent++;
+
+ long now = System.currentTimeMillis();
+ if (now >= lastCheck + 5000) {
+ logger.debug("Packed {} bytes, {} events", sentBytes, sent);
+ lastCheck = now;
+ }
+ }
+ rpcClient.appendBatch(events);
+ if (reader instanceof ReliableEventReader) {
+ ((ReliableEventReader) reader).commit();
+ }
+ }
+ logger.debug("Finished");
+ } finally {
+ if (reader != null) {
+ logger.debug("Closing reader");
+ reader.close();
+ }
+
+ logger.debug("Closing RPC client");
+ rpcClient.close();
}
- }
- logger.debug("Finished");
- } finally {
- if (reader != null) {
- logger.debug("Closing reader");
- reader.close();
- }
-
- logger.debug("Closing RPC client");
- rpcClient.close();
}
- }
}
diff --git a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/client/NettyAvroRpcClient.java b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/client/NettyAvroRpcClient.java
index 9c21093..798c89a 100644
--- a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/client/NettyAvroRpcClient.java
+++ b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/client/NettyAvroRpcClient.java
@@ -1,23 +1,26 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.rpc.avro.client;
+import io.netty.channel.ChannelPipeline;
+import io.netty.handler.codec.compression.JZlibDecoder;
+import io.netty.handler.codec.compression.JZlibEncoder;
+import io.netty.handler.codec.compression.ZlibEncoder;
+import io.netty.handler.ssl.SslHandler;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -51,7 +54,6 @@ import javax.net.ssl.SSLEngine;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
-
import org.apache.avro.ipc.CallFuture;
import org.apache.avro.ipc.Transceiver;
import org.apache.avro.ipc.netty.NettyTransceiver;
@@ -69,12 +71,6 @@ import org.apache.flume.source.avro.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import io.netty.channel.ChannelPipeline;
-import io.netty.handler.codec.compression.JZlibDecoder;
-import io.netty.handler.codec.compression.JZlibEncoder;
-import io.netty.handler.codec.compression.ZlibEncoder;
-import io.netty.handler.ssl.SslHandler;
-
/**
* Avro/Netty implementation of {@link org.apache.flume.rpc.api.RpcClient}.
* The connections are intended to be opened before clients are given access so
@@ -82,576 +78,569 @@ import io.netty.handler.ssl.SslHandler;
*/
public class NettyAvroRpcClient extends SSLContextAwareAbstractRpcClient {
- private ExecutorService callTimeoutPool;
- private final ReentrantLock stateLock = new ReentrantLock();
-
- /**
- * Guarded by {@code stateLock}
- */
- private ConnState connState;
-
- private InetSocketAddress address;
-
- private Transceiver transceiver;
- private AvroSourceProtocol.Callback avroClient;
- private static final Logger logger = LoggerFactory.getLogger(NettyAvroRpcClient.class);
- private boolean enableDeflateCompression;
- private int compressionLevel;
-
- public static NettyAvroRpcClient newInstance() {
- return new NettyAvroRpcClient();
- }
-
- /**
- * This constructor is intended to be called from {@link org.apache.flume.rpc.api.RpcClientFactory}.
- * A call to this constructor should be followed by call to configure().
- */
- protected NettyAvroRpcClient() {
- }
-
- /**
- * This method should only be invoked by the build function
- * @throws FlumeException
- */
- private void connect() throws FlumeException {
- connect(connectTimeout, TimeUnit.MILLISECONDS);
- }
-
- /**
- * Internal only, for now
- * @param timeout
- * @param tu
- * @throws FlumeException
- */
- private void connect(long timeout, TimeUnit tu) throws FlumeException {
- callTimeoutPool = Executors.newCachedThreadPool(
- new TransceiverThreadFactory("Flume Avro RPC Client Call Invoker"));
-
- try {
- transceiver = new NettyTransceiver(this.address, Math.toIntExact(tu.toMillis(timeout)),
- (ch) -> {
+ private ExecutorService callTimeoutPool;
+ private final ReentrantLock stateLock = new ReentrantLock();
+
+ /**
+ * Guarded by {@code stateLock}
+ */
+ private ConnState connState;
+
+ private InetSocketAddress address;
+
+ private Transceiver transceiver;
+ private AvroSourceProtocol.Callback avroClient;
+ private static final Logger logger = LoggerFactory.getLogger(NettyAvroRpcClient.class);
+ private boolean enableDeflateCompression;
+ private int compressionLevel;
+
+ public static NettyAvroRpcClient newInstance() {
+ return new NettyAvroRpcClient();
+ }
+
+ /**
+ * This constructor is intended to be called from {@link org.apache.flume.rpc.api.RpcClientFactory}.
+ * A call to this constructor should be followed by call to configure().
+ */
+ protected NettyAvroRpcClient() {}
+
+ /**
+ * This method should only be invoked by the build function
+ * @throws FlumeException
+ */
+ private void connect() throws FlumeException {
+ connect(connectTimeout, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Internal only, for now
+ * @param timeout
+ * @param tu
+ * @throws FlumeException
+ */
+ private void connect(long timeout, TimeUnit tu) throws FlumeException {
+ callTimeoutPool =
+ Executors.newCachedThreadPool(new TransceiverThreadFactory("Flume Avro RPC Client Call Invoker"));
+
+ try {
+ transceiver = new NettyTransceiver(this.address, Math.toIntExact(tu.toMillis(timeout)), (ch) -> {
ChannelPipeline pipeline = ch.pipeline();
if (enableDeflateCompression) {
- ZlibEncoder encoder = new JZlibEncoder(compressionLevel);
- pipeline.addFirst("deflater", encoder);
- pipeline.addFirst("inflater", new JZlibDecoder());
+ ZlibEncoder encoder = new JZlibEncoder(compressionLevel);
+ pipeline.addFirst("deflater", encoder);
+ pipeline.addFirst("inflater", new JZlibDecoder());
}
SSLEngine engine = createSSLEngine();
if (engine != null) {
- engine.setUseClientMode(true);
- pipeline.addLast("ssl", new SslHandler(engine));
+ engine.setUseClientMode(true);
+ pipeline.addLast("ssl", new SslHandler(engine));
}
- });
- avroClient = SpecificRequestor.getClient(AvroSourceProtocol.Callback.class, transceiver);
- } catch (Throwable t) {
- if (callTimeoutPool != null) {
- callTimeoutPool.shutdownNow();
- }
- if (t instanceof IOException) {
- throw new FlumeException(this + ": RPC connection error", t);
- } else if (t instanceof FlumeException) {
- throw (FlumeException) t;
- } else if (t instanceof Error) {
- throw (Error) t;
- } else {
- throw new FlumeException(this + ": Unexpected exception", t);
- }
+ });
+ avroClient = SpecificRequestor.getClient(AvroSourceProtocol.Callback.class, transceiver);
+ } catch (Throwable t) {
+ if (callTimeoutPool != null) {
+ callTimeoutPool.shutdownNow();
+ }
+ if (t instanceof IOException) {
+ throw new FlumeException(this + ": RPC connection error", t);
+ } else if (t instanceof FlumeException) {
+ throw (FlumeException) t;
+ } else if (t instanceof Error) {
+ throw (Error) t;
+ } else {
+ throw new FlumeException(this + ": Unexpected exception", t);
+ }
+ }
+
+ setState(ConnState.READY);
}
- setState(ConnState.READY);
- }
-
- @Override
- public void close() throws FlumeException {
- if (callTimeoutPool != null) {
- callTimeoutPool.shutdown();
- try {
- if (!callTimeoutPool.awaitTermination(requestTimeout, TimeUnit.MILLISECONDS)) {
- callTimeoutPool.shutdownNow();
- if (!callTimeoutPool.awaitTermination(requestTimeout, TimeUnit.MILLISECONDS)) {
- logger.warn(this + ": Unable to cleanly shut down call timeout pool");
- }
+ @Override
+ public void close() throws FlumeException {
+ if (callTimeoutPool != null) {
+ callTimeoutPool.shutdown();
+ try {
+ if (!callTimeoutPool.awaitTermination(requestTimeout, TimeUnit.MILLISECONDS)) {
+ callTimeoutPool.shutdownNow();
+ if (!callTimeoutPool.awaitTermination(requestTimeout, TimeUnit.MILLISECONDS)) {
+ logger.warn(this + ": Unable to cleanly shut down call timeout pool");
+ }
+ }
+ } catch (InterruptedException ex) {
+ logger.warn(this + ": Interrupted during close", ex);
+ // re-cancel if current thread also interrupted
+ callTimeoutPool.shutdownNow();
+ // preserve interrupt status
+ Thread.currentThread().interrupt();
+ }
+
+ callTimeoutPool = null;
+ }
+ try {
+ if (transceiver != null) {
+ transceiver.close();
+ }
+ } catch (IOException ex) {
+ throw new FlumeException(this + ": Error closing transceiver.", ex);
+ } finally {
+ setState(ConnState.DEAD);
}
- } catch (InterruptedException ex) {
- logger.warn(this + ": Interrupted during close", ex);
- // re-cancel if current thread also interrupted
- callTimeoutPool.shutdownNow();
- // preserve interrupt status
- Thread.currentThread().interrupt();
- }
-
- callTimeoutPool = null;
}
- try {
- if (transceiver != null) {
- transceiver.close();
- }
- } catch (IOException ex) {
- throw new FlumeException(this + ": Error closing transceiver.", ex);
- } finally {
- setState(ConnState.DEAD);
+
+ @Override
+ public String toString() {
+ return "NettyAvroRpcClient { host: " + address.getHostName() + ", port: " + address.getPort() + " }";
}
- }
-
- @Override
- public String toString() {
- return "NettyAvroRpcClient { host: " + address.getHostName() + ", port: " + address.getPort() + " }";
- }
-
- @Override
- public void append(Event event) throws EventDeliveryException {
- try {
- append(event, requestTimeout, TimeUnit.MILLISECONDS);
- } catch (Throwable t) {
- // we mark as no longer active without trying to clean up resources
- // client is required to call close() to clean up resources
- setState(ConnState.DEAD);
- if (t instanceof Error) {
- throw (Error) t;
- }
- if (t instanceof TimeoutException) {
- throw new EventDeliveryException(this + ": Failed to send event. " +
- "RPC request timed out after " + requestTimeout + "ms", t);
- }
- throw new EventDeliveryException(this + ": Failed to send event", t);
+ @Override
+ public void append(Event event) throws EventDeliveryException {
+ try {
+ append(event, requestTimeout, TimeUnit.MILLISECONDS);
+ } catch (Throwable t) {
+ // we mark as no longer active without trying to clean up resources
+ // client is required to call close() to clean up resources
+ setState(ConnState.DEAD);
+ if (t instanceof Error) {
+ throw (Error) t;
+ }
+ if (t instanceof TimeoutException) {
+ throw new EventDeliveryException(
+ this + ": Failed to send event. " + "RPC request timed out after " + requestTimeout + "ms", t);
+ }
+ throw new EventDeliveryException(this + ": Failed to send event", t);
+ }
}
- }
- private void append(Event event, long timeout, TimeUnit tu)
- throws EventDeliveryException {
+ private void append(Event event, long timeout, TimeUnit tu) throws EventDeliveryException {
- assertReady();
+ assertReady();
- final CallFuture<Status> callFuture = new CallFuture<Status>();
+ final CallFuture<Status> callFuture = new CallFuture<Status>();
- final AvroFlumeEvent avroEvent = new AvroFlumeEvent();
- avroEvent.setBody(ByteBuffer.wrap(event.getBody()));
- avroEvent.setHeaders(toCharSeqMap(event.getHeaders()));
+ final AvroFlumeEvent avroEvent = new AvroFlumeEvent();
+ avroEvent.setBody(ByteBuffer.wrap(event.getBody()));
+ avroEvent.setHeaders(toCharSeqMap(event.getHeaders()));
- Future<Void> handshake;
- try {
- // due to AVRO-1122, avroClient.append() may block
- handshake = callTimeoutPool.submit(new Callable<Void>() {
+ Future<Void> handshake;
+ try {
+ // due to AVRO-1122, avroClient.append() may block
+ handshake = callTimeoutPool.submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- avroClient.append(avroEvent, callFuture);
- return null;
+ @Override
+ public Void call() throws Exception {
+ avroClient.append(avroEvent, callFuture);
+ return null;
+ }
+ });
+ } catch (RejectedExecutionException ex) {
+ throw new EventDeliveryException(this + ": Executor error", ex);
+ }
+
+ try {
+ handshake.get(connectTimeout, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException ex) {
+ throw new EventDeliveryException(this + ": Handshake timed out after " + connectTimeout + " ms", ex);
+ } catch (InterruptedException ex) {
+ throw new EventDeliveryException(this + ": Interrupted in handshake", ex);
+ } catch (ExecutionException ex) {
+ throw new EventDeliveryException(this + ": RPC request exception", ex);
+ } catch (CancellationException ex) {
+ throw new EventDeliveryException(this + ": RPC request cancelled", ex);
+ } finally {
+ if (!handshake.isDone()) {
+ handshake.cancel(true);
+ }
}
- });
- } catch (RejectedExecutionException ex) {
- throw new EventDeliveryException(this + ": Executor error", ex);
- }
- try {
- handshake.get(connectTimeout, TimeUnit.MILLISECONDS);
- } catch (TimeoutException ex) {
- throw new EventDeliveryException(this + ": Handshake timed out after " + connectTimeout + " ms", ex);
- } catch (InterruptedException ex) {
- throw new EventDeliveryException(this + ": Interrupted in handshake", ex);
- } catch (ExecutionException ex) {
- throw new EventDeliveryException(this + ": RPC request exception", ex);
- } catch (CancellationException ex) {
- throw new EventDeliveryException(this + ": RPC request cancelled", ex);
- } finally {
- if (!handshake.isDone()) {
- handshake.cancel(true);
- }
+ waitForStatusOK(callFuture, timeout, tu);
}
- waitForStatusOK(callFuture, timeout, tu);
- }
-
- @Override
- public void appendBatch(List<Event> events) throws EventDeliveryException {
- try {
- appendBatch(events, requestTimeout, TimeUnit.MILLISECONDS);
- } catch (Throwable t) {
- // we mark as no longer active without trying to clean up resources
- // client is required to call close() to clean up resources
- setState(ConnState.DEAD);
- if (t instanceof Error) {
- throw (Error) t;
- }
- if (t instanceof TimeoutException) {
- throw new EventDeliveryException(this + ": Failed to send event. " +
- "RPC request timed out after " + requestTimeout + " ms", t);
- }
- throw new EventDeliveryException(this + ": Failed to send batch", t);
+ @Override
+ public void appendBatch(List<Event> events) throws EventDeliveryException {
+ try {
+ appendBatch(events, requestTimeout, TimeUnit.MILLISECONDS);
+ } catch (Throwable t) {
+ // we mark as no longer active without trying to clean up resources
+ // client is required to call close() to clean up resources
+ setState(ConnState.DEAD);
+ if (t instanceof Error) {
+ throw (Error) t;
+ }
+ if (t instanceof TimeoutException) {
+ throw new EventDeliveryException(
+ this + ": Failed to send event. " + "RPC request timed out after " + requestTimeout + " ms", t);
+ }
+ throw new EventDeliveryException(this + ": Failed to send batch", t);
+ }
}
- }
- private void appendBatch(List<Event> events, long timeout, TimeUnit tu)
- throws EventDeliveryException {
+ private void appendBatch(List<Event> events, long timeout, TimeUnit tu) throws EventDeliveryException {
- assertReady();
+ assertReady();
- Iterator<Event> iter = events.iterator();
- final List<AvroFlumeEvent> avroEvents = new LinkedList<AvroFlumeEvent>();
+ Iterator<Event> iter = events.iterator();
+ final List<AvroFlumeEvent> avroEvents = new LinkedList<AvroFlumeEvent>();
- // send multiple batches... bail if there is a problem at any time
- while (iter.hasNext()) {
- avroEvents.clear();
+ // send multiple batches... bail if there is a problem at any time
+ while (iter.hasNext()) {
+ avroEvents.clear();
- for (int i = 0; i < batchSize && iter.hasNext(); i++) {
- Event event = iter.next();
- AvroFlumeEvent avroEvent = new AvroFlumeEvent();
- avroEvent.setBody(ByteBuffer.wrap(event.getBody()));
- avroEvent.setHeaders(toCharSeqMap(event.getHeaders()));
- avroEvents.add(avroEvent);
- }
-
- final CallFuture<Status> callFuture = new CallFuture<Status>();
-
- Future<Void> handshake;
- try {
- // due to AVRO-1122, avroClient.appendBatch() may block
- handshake = callTimeoutPool.submit(new Callable<Void>() {
-
- @Override
- public Void call() throws Exception {
- avroClient.appendBatch(avroEvents, callFuture);
- return null;
- }
- });
- } catch (RejectedExecutionException ex) {
- throw new EventDeliveryException(this + ": Executor error", ex);
- }
-
- try {
- handshake.get(connectTimeout, TimeUnit.MILLISECONDS);
- } catch (TimeoutException ex) {
- throw new EventDeliveryException(this + ": Handshake timed out after " +
- connectTimeout + "ms", ex);
- } catch (InterruptedException ex) {
- throw new EventDeliveryException(this + ": Interrupted in handshake",
- ex);
- } catch (ExecutionException ex) {
- throw new EventDeliveryException(this + ": RPC request exception", ex);
- } catch (CancellationException ex) {
- throw new EventDeliveryException(this + ": RPC request cancelled", ex);
- } finally {
- if (!handshake.isDone()) {
- handshake.cancel(true);
- }
- }
+ for (int i = 0; i < batchSize && iter.hasNext(); i++) {
+ Event event = iter.next();
+ AvroFlumeEvent avroEvent = new AvroFlumeEvent();
+ avroEvent.setBody(ByteBuffer.wrap(event.getBody()));
+ avroEvent.setHeaders(toCharSeqMap(event.getHeaders()));
+ avroEvents.add(avroEvent);
+ }
- waitForStatusOK(callFuture, timeout, tu);
- }
- }
-
- /**
- * Helper method that waits for a Status future to come back and validates
- * that it returns Status == OK.
- * @param callFuture Future to wait on
- * @param timeout Time to wait before failing
- * @param tu Time Unit of {@code timeout}
- * @throws EventDeliveryException If there is a timeout or if Status != OK
- */
- private void waitForStatusOK(CallFuture<Status> callFuture,
- long timeout, TimeUnit tu) throws EventDeliveryException {
- try {
- Status status = callFuture.get(timeout, tu);
- if (status != Status.OK) {
- throw new EventDeliveryException(this + ": Avro RPC call returned Status: " + status);
- }
- } catch (CancellationException ex) {
- throw new EventDeliveryException(this + ": RPC future was cancelled", ex);
- } catch (ExecutionException ex) {
- throw new EventDeliveryException(this + ": Exception thrown from remote handler", ex);
- } catch (TimeoutException ex) {
- throw new EventDeliveryException(this + ": RPC request timed out", ex);
- } catch (InterruptedException ex) {
- Thread.currentThread().interrupt();
- throw new EventDeliveryException(this + ": RPC request interrupted", ex);
- }
- }
-
- /**
- * This method should always be used to change {@code connState} so we ensure
- * that invalid state transitions do not occur and that the {@code isIdle}
- * {@link Condition} variable gets signaled reliably.
- * Throws {@code IllegalStateException} when called to transition from CLOSED
- * to another state.
- * @param newState
- */
- private void setState(ConnState newState) {
- stateLock.lock();
- try {
- if (connState == ConnState.DEAD && connState != newState) {
- throw new IllegalStateException("Cannot transition from CLOSED state.");
- }
- connState = newState;
- } finally {
- stateLock.unlock();
- }
- }
-
- /**
- * If the connection state != READY, throws {@link EventDeliveryException}.
- */
- private void assertReady() throws EventDeliveryException {
- stateLock.lock();
- try {
- ConnState curState = connState;
- if (curState != ConnState.READY) {
- throw new EventDeliveryException("RPC failed, client in an invalid state: " + curState);
- }
- } finally {
- stateLock.unlock();
- }
- }
-
- /**
- * Helper function to convert a map of String to a map of CharSequence.
- */
- private static Map<CharSequence, CharSequence> toCharSeqMap(Map<String, String> stringMap) {
- Map<CharSequence, CharSequence> charSeqMap = new HashMap<>();
- for (Map.Entry<String, String> entry : stringMap.entrySet()) {
- charSeqMap.put(entry.getKey(), entry.getValue());
- }
- return charSeqMap;
- }
-
- @Override
- public boolean isActive() {
- stateLock.lock();
- try {
- return (connState == ConnState.READY);
- } finally {
- stateLock.unlock();
- }
- }
-
- private static enum ConnState {
- INIT, READY, DEAD
- }
-
- /**
- * <p>
- * Configure the actual client using the properties.
- * <tt>properties</tt> should have at least 2 params:
- * <p><tt>hosts</tt> = <i>alias_for_host</i></p>
- * <p><tt>alias_for_host</tt> = <i>hostname:port</i>. </p>
- * Only the first host is added, rest are discarded.</p>
- * <p>Optionally it can also have a <p>
- * <tt>batch-size</tt> = <i>batchSize</i>
- * @param properties The properties to instantiate the client with.
- * @return
- */
- @Override
- public synchronized void configure(Properties properties) throws FlumeException {
- stateLock.lock();
- try {
- if (connState == ConnState.READY || connState == ConnState.DEAD) {
- throw new FlumeException("This client was already configured, cannot reconfigure.");
- }
- } finally {
- stateLock.unlock();
- }
+ final CallFuture<Status> callFuture = new CallFuture<Status>();
+
+ Future<Void> handshake;
+ try {
+ // due to AVRO-1122, avroClient.appendBatch() may block
+ handshake = callTimeoutPool.submit(new Callable<Void>() {
+
+ @Override
+ public Void call() throws Exception {
+ avroClient.appendBatch(avroEvents, callFuture);
+ return null;
+ }
+ });
+ } catch (RejectedExecutionException ex) {
+ throw new EventDeliveryException(this + ": Executor error", ex);
+ }
- batchSize = AbstractRpcClient.parseBatchSize(properties);
+ try {
+ handshake.get(connectTimeout, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException ex) {
+ throw new EventDeliveryException(this + ": Handshake timed out after " + connectTimeout + "ms", ex);
+ } catch (InterruptedException ex) {
+ throw new EventDeliveryException(this + ": Interrupted in handshake", ex);
+ } catch (ExecutionException ex) {
+ throw new EventDeliveryException(this + ": RPC request exception", ex);
+ } catch (CancellationException ex) {
+ throw new EventDeliveryException(this + ": RPC request cancelled", ex);
+ } finally {
+ if (!handshake.isDone()) {
+ handshake.cancel(true);
+ }
+ }
- // host and port
- String hostNames = properties.getProperty(RpcClientConfigurationConstants.CONFIG_HOSTS);
- String[] hosts = null;
- if (hostNames != null && !hostNames.isEmpty()) {
- hosts = hostNames.split("\\s+");
- } else {
- throw new FlumeException("Hosts list is invalid: " + hostNames);
+ waitForStatusOK(callFuture, timeout, tu);
+ }
}
- if (hosts.length > 1) {
- logger.warn("More than one hosts are specified for the default client. "
- + "Only the first host will be used and others ignored. Specified: "
- + hostNames + "; to be used: " + hosts[0]);
+ /**
+ * Helper method that waits for a Status future to come back and validates
+ * that it returns Status == OK.
+ * @param callFuture Future to wait on
+ * @param timeout Time to wait before failing
+ * @param tu Time Unit of {@code timeout}
+ * @throws EventDeliveryException If there is a timeout or if Status != OK
+ */
+ private void waitForStatusOK(CallFuture<Status> callFuture, long timeout, TimeUnit tu)
+ throws EventDeliveryException {
+ try {
+ Status status = callFuture.get(timeout, tu);
+ if (status != Status.OK) {
+ throw new EventDeliveryException(this + ": Avro RPC call returned Status: " + status);
+ }
+ } catch (CancellationException ex) {
+ throw new EventDeliveryException(this + ": RPC future was cancelled", ex);
+ } catch (ExecutionException ex) {
+ throw new EventDeliveryException(this + ": Exception thrown from remote handler", ex);
+ } catch (TimeoutException ex) {
+ throw new EventDeliveryException(this + ": RPC request timed out", ex);
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new EventDeliveryException(this + ": RPC request interrupted", ex);
+ }
}
- String host = properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + hosts[0]);
- if (host == null || host.isEmpty()) {
- throw new FlumeException("Host not found: " + hosts[0]);
- }
- String[] hostAndPort = host.split(":");
- if (hostAndPort.length != 2) {
- throw new FlumeException("Invalid hostname: " + hosts[0]);
- }
- Integer port = null;
- try {
- port = Integer.parseInt(hostAndPort[1]);
- } catch (NumberFormatException e) {
- throw new FlumeException("Invalid Port: " + hostAndPort[1], e);
+ /**
+ * This method should always be used to change {@code connState} so we ensure
+ * that invalid state transitions do not occur and that the {@code isIdle}
+ * {@link Condition} variable gets signaled reliably.
+ * Throws {@code IllegalStateException} when called to transition from CLOSED
+ * to another state.
+ * @param newState
+ */
+ private void setState(ConnState newState) {
+ stateLock.lock();
+ try {
+ if (connState == ConnState.DEAD && connState != newState) {
+ throw new IllegalStateException("Cannot transition from CLOSED state.");
+ }
+ connState = newState;
+ } finally {
+ stateLock.unlock();
+ }
}
- this.address = new InetSocketAddress(hostAndPort[0], port);
-
- // connect timeout
- connectTimeout = RpcClientConfigurationConstants.DEFAULT_CONNECT_TIMEOUT_MILLIS;
- String strConnTimeout = properties.getProperty(
- RpcClientConfigurationConstants.CONFIG_CONNECT_TIMEOUT);
- if (strConnTimeout != null && strConnTimeout.trim().length() > 0) {
- try {
- connectTimeout = Long.parseLong(strConnTimeout);
- if (connectTimeout < 1000) {
- logger.warn("Connection timeout specified less than 1s. Using default value instead.");
- connectTimeout = RpcClientConfigurationConstants.DEFAULT_CONNECT_TIMEOUT_MILLIS;
+
+ /**
+ * If the connection state != READY, throws {@link EventDeliveryException}.
+ */
+ private void assertReady() throws EventDeliveryException {
+ stateLock.lock();
+ try {
+ ConnState curState = connState;
+ if (curState != ConnState.READY) {
+ throw new EventDeliveryException("RPC failed, client in an invalid state: " + curState);
+ }
+ } finally {
+ stateLock.unlock();
}
- } catch (NumberFormatException ex) {
- logger.error("Invalid connect timeout specified: " + strConnTimeout);
- }
}
- // request timeout
- requestTimeout = RpcClientConfigurationConstants.DEFAULT_REQUEST_TIMEOUT_MILLIS;
- String strReqTimeout = properties.getProperty(RpcClientConfigurationConstants.CONFIG_REQUEST_TIMEOUT);
- if (strReqTimeout != null && strReqTimeout.trim().length() > 0) {
- try {
- requestTimeout = Long.parseLong(strReqTimeout);
- if (requestTimeout < 1000) {
- logger.warn("Request timeout specified less than 1s. Using default value instead.");
- requestTimeout = RpcClientConfigurationConstants.DEFAULT_REQUEST_TIMEOUT_MILLIS;
+ /**
+ * Helper function to convert a map of String to a map of CharSequence.
+ */
+ private static Map<CharSequence, CharSequence> toCharSeqMap(Map<String, String> stringMap) {
+ Map<CharSequence, CharSequence> charSeqMap = new HashMap<>();
+ for (Map.Entry<String, String> entry : stringMap.entrySet()) {
+ charSeqMap.put(entry.getKey(), entry.getValue());
}
- } catch (NumberFormatException ex) {
- logger.error("Invalid request timeout specified: " + strReqTimeout);
- }
+ return charSeqMap;
}
- String enableCompressionStr =
- properties.getProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE);
- if (enableCompressionStr != null && enableCompressionStr.equalsIgnoreCase("deflate")) {
- this.enableDeflateCompression = true;
- String compressionLvlStr =
- properties.getProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_LEVEL);
- compressionLevel = RpcClientConfigurationConstants.DEFAULT_COMPRESSION_LEVEL;
- if (compressionLvlStr != null) {
+ @Override
+ public boolean isActive() {
+ stateLock.lock();
try {
- compressionLevel = Integer.parseInt(compressionLvlStr);
- } catch (NumberFormatException ex) {
- logger.error("Invalid compression level: " + compressionLvlStr);
+ return (connState == ConnState.READY);
+ } finally {
+ stateLock.unlock();
}
- }
}
- configureSSL(properties);
-
- String maxIoWorkersStr = properties.getProperty(RpcClientConfigurationConstants.MAX_IO_WORKERS);
- if (!StringUtils.isEmpty(maxIoWorkersStr)) {
- logger.warn("Specifying the number of workers is no longer supported");
+ private static enum ConnState {
+ INIT,
+ READY,
+ DEAD
}
- this.connect();
- }
-
- /**
- * A thread factor implementation modeled after the implementation of
- * NettyTransceiver.NettyTransceiverThreadFactory class which is
- * a private static class. The only difference between that and this
- * implementation is that this implementation marks all the threads daemon
- * which allows the termination of the VM when the non-daemon threads
- * are done.
- */
- private static class TransceiverThreadFactory implements ThreadFactory {
- private final AtomicInteger threadId = new AtomicInteger(0);
- private final String prefix;
-
/**
- * Creates a TransceiverThreadFactory that creates threads with the
- * specified name.
- * @param prefix the name prefix to use for all threads created by this
- * ThreadFactory. A unique ID will be appended to this prefix to form the
- * final thread name.
+ * <p>
+ * Configure the actual client using the properties.
+ * <tt>properties</tt> should have at least 2 params:
+ * <p><tt>hosts</tt> = <i>alias_for_host</i></p>
+ * <p><tt>alias_for_host</tt> = <i>hostname:port</i>. </p>
+ * Only the first host is added, rest are discarded.</p>
+ * <p>Optionally it can also have a <p>
+ * <tt>batch-size</tt> = <i>batchSize</i>
+ * @param properties The properties to instantiate the client with.
+ * @return
*/
- public TransceiverThreadFactory(String prefix) {
- this.prefix = prefix;
- }
-
@Override
- public Thread newThread(Runnable r) {
- Thread thread = new Thread(r);
- thread.setDaemon(true);
- thread.setName(prefix + " " + threadId.incrementAndGet());
- return thread;
- }
- }
-
- private SSLEngine createSSLEngine() {
- TrustManager[] managers;
- try {
- if (enableSsl) {
- if (trustAllCerts) {
- logger.warn(
- "No truststore configured, setting TrustManager to accept all server certificates");
- managers = new TrustManager[]{new PermissiveTrustManager()};
+ public synchronized void configure(Properties properties) throws FlumeException {
+ stateLock.lock();
+ try {
+ if (connState == ConnState.READY || connState == ConnState.DEAD) {
+ throw new FlumeException("This client was already configured, cannot reconfigure.");
+ }
+ } finally {
+ stateLock.unlock();
+ }
+
+ batchSize = AbstractRpcClient.parseBatchSize(properties);
+
+ // host and port
+ String hostNames = properties.getProperty(RpcClientConfigurationConstants.CONFIG_HOSTS);
+ String[] hosts = null;
+ if (hostNames != null && !hostNames.isEmpty()) {
+ hosts = hostNames.split("\\s+");
} else {
- KeyStore keystore = null;
+ throw new FlumeException("Hosts list is invalid: " + hostNames);
+ }
- if (truststore != null) {
- try (InputStream truststoreStream = new FileInputStream(truststore)) {
- keystore = KeyStore.getInstance(truststoreType);
- keystore.load(truststoreStream, truststorePassword != null ? truststorePassword.toCharArray() : null);
+ if (hosts.length > 1) {
+ logger.warn("More than one hosts are specified for the default client. "
+ + "Only the first host will be used and others ignored. Specified: "
+ + hostNames + "; to be used: " + hosts[0]);
+ }
+
+ String host = properties.getProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + hosts[0]);
+ if (host == null || host.isEmpty()) {
+ throw new FlumeException("Host not found: " + hosts[0]);
+ }
+ String[] hostAndPort = host.split(":");
+ if (hostAndPort.length != 2) {
+ throw new FlumeException("Invalid hostname: " + hosts[0]);
+ }
+ Integer port = null;
+ try {
+ port = Integer.parseInt(hostAndPort[1]);
+ } catch (NumberFormatException e) {
+ throw new FlumeException("Invalid Port: " + hostAndPort[1], e);
+ }
+ this.address = new InetSocketAddress(hostAndPort[0], port);
+
+ // connect timeout
+ connectTimeout = RpcClientConfigurationConstants.DEFAULT_CONNECT_TIMEOUT_MILLIS;
+ String strConnTimeout = properties.getProperty(RpcClientConfigurationConstants.CONFIG_CONNECT_TIMEOUT);
+ if (strConnTimeout != null && strConnTimeout.trim().length() > 0) {
+ try {
+ connectTimeout = Long.parseLong(strConnTimeout);
+ if (connectTimeout < 1000) {
+ logger.warn("Connection timeout specified less than 1s. Using default value instead.");
+ connectTimeout = RpcClientConfigurationConstants.DEFAULT_CONNECT_TIMEOUT_MILLIS;
+ }
+ } catch (NumberFormatException ex) {
+ logger.error("Invalid connect timeout specified: " + strConnTimeout);
}
- }
+ }
- TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
- // null keystore is OK, with SunX509 it defaults to system CA Certs
- // see http://docs.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#X509TrustManager
- tmf.init(keystore);
- managers = tmf.getTrustManagers();
+ // request timeout
+ requestTimeout = RpcClientConfigurationConstants.DEFAULT_REQUEST_TIMEOUT_MILLIS;
+ String strReqTimeout = properties.getProperty(RpcClientConfigurationConstants.CONFIG_REQUEST_TIMEOUT);
+ if (strReqTimeout != null && strReqTimeout.trim().length() > 0) {
+ try {
+ requestTimeout = Long.parseLong(strReqTimeout);
+ if (requestTimeout < 1000) {
+ logger.warn("Request timeout specified less than 1s. Using default value instead.");
+ requestTimeout = RpcClientConfigurationConstants.DEFAULT_REQUEST_TIMEOUT_MILLIS;
+ }
+ } catch (NumberFormatException ex) {
+ logger.error("Invalid request timeout specified: " + strReqTimeout);
+ }
}
- SSLContext sslContext = SSLContext.getInstance("TLS");
- sslContext.init(null, managers, null);
- SSLEngine sslEngine = sslContext.createSSLEngine();
- sslEngine.setUseClientMode(true);
-
- List<String> enabledProtocols = new ArrayList<String>();
- for (String protocol : sslEngine.getEnabledProtocols()) {
- if ((includeProtocols.isEmpty() || includeProtocols.contains(protocol))
- && !excludeProtocols.contains(protocol)) {
- enabledProtocols.add(protocol);
- }
+ String enableCompressionStr = properties.getProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE);
+ if (enableCompressionStr != null && enableCompressionStr.equalsIgnoreCase("deflate")) {
+ this.enableDeflateCompression = true;
+ String compressionLvlStr = properties.getProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_LEVEL);
+ compressionLevel = RpcClientConfigurationConstants.DEFAULT_COMPRESSION_LEVEL;
+ if (compressionLvlStr != null) {
+ try {
+ compressionLevel = Integer.parseInt(compressionLvlStr);
+ } catch (NumberFormatException ex) {
+ logger.error("Invalid compression level: " + compressionLvlStr);
+ }
+ }
}
- sslEngine.setEnabledProtocols(enabledProtocols.toArray(new String[0]));
-
- List<String> enabledCipherSuites = new ArrayList<String>();
- for (String suite : sslEngine.getEnabledCipherSuites()) {
- if ((includeCipherSuites.isEmpty() || includeCipherSuites.contains(suite))
- && !excludeCipherSuites.contains(suite)) {
- enabledCipherSuites.add(suite);
- }
+
+ configureSSL(properties);
+
+ String maxIoWorkersStr = properties.getProperty(RpcClientConfigurationConstants.MAX_IO_WORKERS);
+ if (!StringUtils.isEmpty(maxIoWorkersStr)) {
+ logger.warn("Specifying the number of workers is no longer supported");
}
- sslEngine.setEnabledCipherSuites(enabledCipherSuites.toArray(new String[0]));
-
- logger.info("SSLEngine protocols enabled: " + Arrays.asList(sslEngine.getEnabledProtocols()));
- logger.info("SSLEngine cipher suites enabled: " + Arrays.asList(sslEngine.getEnabledProtocols()));
- return sslEngine;
- } else {
- return null;
- }
-
- } catch (Exception ex) {
- logger.error("Cannot create SSL channel", ex);
- throw new RuntimeException("Cannot create SSL channel", ex);
+
+ this.connect();
}
- }
- /**
- * Permissive trust manager accepting any certificate
- */
- private static class PermissiveTrustManager implements X509TrustManager {
- @Override
- public void checkClientTrusted(X509Certificate[] certs, String s) {
- // nothing
+ /**
+ * A thread factor implementation modeled after the implementation of
+ * NettyTransceiver.NettyTransceiverThreadFactory class which is
+ * a private static class. The only difference between that and this
+ * implementation is that this implementation marks all the threads daemon
+ * which allows the termination of the VM when the non-daemon threads
+ * are done.
+ */
+ private static class TransceiverThreadFactory implements ThreadFactory {
+ private final AtomicInteger threadId = new AtomicInteger(0);
+ private final String prefix;
+
+ /**
+ * Creates a TransceiverThreadFactory that creates threads with the
+ * specified name.
+ * @param prefix the name prefix to use for all threads created by this
+ * ThreadFactory. A unique ID will be appended to this prefix to form the
+ * final thread name.
+ */
+ public TransceiverThreadFactory(String prefix) {
+ this.prefix = prefix;
+ }
+
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread thread = new Thread(r);
+ thread.setDaemon(true);
+ thread.setName(prefix + " " + threadId.incrementAndGet());
+ return thread;
+ }
}
- @Override
- public void checkServerTrusted(X509Certificate[] certs, String s) {
- // nothing
+ private SSLEngine createSSLEngine() {
+ TrustManager[] managers;
+ try {
+ if (enableSsl) {
+ if (trustAllCerts) {
+ logger.warn("No truststore configured, setting TrustManager to accept all server certificates");
+ managers = new TrustManager[] {new PermissiveTrustManager()};
+ } else {
+ KeyStore keystore = null;
+
+ if (truststore != null) {
+ try (InputStream truststoreStream = new FileInputStream(truststore)) {
+ keystore = KeyStore.getInstance(truststoreType);
+ keystore.load(
+ truststoreStream,
+ truststorePassword != null ? truststorePassword.toCharArray() : null);
+ }
+ }
+
+ TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
+ // null keystore is OK, with SunX509 it defaults to system CA Certs
+ // see
+ // http://docs.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#X509TrustManager
+ tmf.init(keystore);
+ managers = tmf.getTrustManagers();
+ }
+
+ SSLContext sslContext = SSLContext.getInstance("TLS");
+ sslContext.init(null, managers, null);
+ SSLEngine sslEngine = sslContext.createSSLEngine();
+ sslEngine.setUseClientMode(true);
+
+ List<String> enabledProtocols = new ArrayList<String>();
+ for (String protocol : sslEngine.getEnabledProtocols()) {
+ if ((includeProtocols.isEmpty() || includeProtocols.contains(protocol))
+ && !excludeProtocols.contains(protocol)) {
+ enabledProtocols.add(protocol);
+ }
+ }
+ sslEngine.setEnabledProtocols(enabledProtocols.toArray(new String[0]));
+
+ List<String> enabledCipherSuites = new ArrayList<String>();
+ for (String suite : sslEngine.getEnabledCipherSuites()) {
+ if ((includeCipherSuites.isEmpty() || includeCipherSuites.contains(suite))
+ && !excludeCipherSuites.contains(suite)) {
+ enabledCipherSuites.add(suite);
+ }
+ }
+ sslEngine.setEnabledCipherSuites(enabledCipherSuites.toArray(new String[0]));
+
+ logger.info("SSLEngine protocols enabled: " + Arrays.asList(sslEngine.getEnabledProtocols()));
+ logger.info("SSLEngine cipher suites enabled: " + Arrays.asList(sslEngine.getEnabledProtocols()));
+ return sslEngine;
+ } else {
+ return null;
+ }
+
+ } catch (Exception ex) {
+ logger.error("Cannot create SSL channel", ex);
+ throw new RuntimeException("Cannot create SSL channel", ex);
+ }
}
- @Override
- public X509Certificate[] getAcceptedIssuers() {
- return new X509Certificate[0];
+ /**
+ * Permissive trust manager accepting any certificate
+ */
+ private static class PermissiveTrustManager implements X509TrustManager {
+ @Override
+ public void checkClientTrusted(X509Certificate[] certs, String s) {
+ // nothing
+ }
+
+ @Override
+ public void checkServerTrusted(X509Certificate[] certs, String s) {
+ // nothing
+ }
+
+ @Override
+ public X509Certificate[] getAcceptedIssuers() {
+ return new X509Certificate[0];
+ }
}
- }
}
diff --git a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AbstractAvroEventSerializer.java b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AbstractAvroEventSerializer.java
index 5becc2d..798da55 100644
--- a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AbstractAvroEventSerializer.java
+++ b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AbstractAvroEventSerializer.java
@@ -1,26 +1,28 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.rpc.avro.serialization;
+import static org.apache.flume.rpc.avro.serialization.AvroEventSerializerConfigurationConstants.COMPRESSION_CODEC;
+import static org.apache.flume.rpc.avro.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_COMPRESSION_CODEC;
+import static org.apache.flume.rpc.avro.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_SYNC_INTERVAL_BYTES;
+import static org.apache.flume.rpc.avro.serialization.AvroEventSerializerConfigurationConstants.SYNC_INTERVAL_BYTES;
+
import java.io.IOException;
import java.io.OutputStream;
-
import org.apache.avro.AvroRuntimeException;
import org.apache.avro.Schema;
import org.apache.avro.file.CodecFactory;
@@ -30,102 +32,92 @@ import org.apache.avro.reflect.ReflectDatumWriter;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.conf.Configurable;
-
import org.apache.flume.serialization.EventSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.flume.rpc.avro.serialization.AvroEventSerializerConfigurationConstants.COMPRESSION_CODEC;
-import static org.apache.flume.rpc.avro.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_COMPRESSION_CODEC;
-import static org.apache.flume.rpc.avro.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_SYNC_INTERVAL_BYTES;
-import static org.apache.flume.rpc.avro.serialization.AvroEventSerializerConfigurationConstants.SYNC_INTERVAL_BYTES;
-
/**
* This is a helper class provided to make it straightforward to serialize
* Flume {@linkplain Event events} into Avro data.
* @param <T> Data type that can be written in the Schema given below.
*/
-public abstract class AbstractAvroEventSerializer<T>
- implements EventSerializer, Configurable {
-
- private static final Logger logger =
- LoggerFactory.getLogger(AbstractAvroEventSerializer.class);
-
- private DatumWriter<T> writer = null;
- private DataFileWriter<T> dataFileWriter = null;
-
- /**
- * Returns the stream to serialize data into.
- */
- protected abstract OutputStream getOutputStream();
-
- /**
- * Returns the parsed Avro schema corresponding to the data being written
- * and the parameterized type specified.
- */
- protected abstract Schema getSchema();
-
- /**
- * Simple conversion routine used to convert an Event to a type of your
- * choosing. That type must correspond to the Avro schema given by
- * {@link #getSchema()}.
- */
- protected abstract T convert(Event event);
-
- @Override
- public void configure(Context context) {
-
- int syncIntervalBytes =
- context.getInteger(SYNC_INTERVAL_BYTES, DEFAULT_SYNC_INTERVAL_BYTES);
- String compressionCodec =
- context.getString(COMPRESSION_CODEC, DEFAULT_COMPRESSION_CODEC);
-
- writer = new ReflectDatumWriter<T>(getSchema());
- dataFileWriter = new DataFileWriter<T>(writer);
-
- dataFileWriter.setSyncInterval(syncIntervalBytes);
-
- try {
- CodecFactory codecFactory = CodecFactory.fromString(compressionCodec);
- dataFileWriter.setCodec(codecFactory);
- } catch (AvroRuntimeException e) {
- logger.warn("Unable to instantiate avro codec with name (" +
- compressionCodec + "). Compression disabled. Exception follows.", e);
+public abstract class AbstractAvroEventSerializer<T> implements EventSerializer, Configurable {
+
+ private static final Logger logger = LoggerFactory.getLogger(AbstractAvroEventSerializer.class);
+
+ private DatumWriter<T> writer = null;
+ private DataFileWriter<T> dataFileWriter = null;
+
+ /**
+ * Returns the stream to serialize data into.
+ */
+ protected abstract OutputStream getOutputStream();
+
+ /**
+ * Returns the parsed Avro schema corresponding to the data being written
+ * and the parameterized type specified.
+ */
+ protected abstract Schema getSchema();
+
+ /**
+ * Simple conversion routine used to convert an Event to a type of your
+ * choosing. That type must correspond to the Avro schema given by
+ * {@link #getSchema()}.
+ */
+ protected abstract T convert(Event event);
+
+ @Override
+ public void configure(Context context) {
+
+ int syncIntervalBytes = context.getInteger(SYNC_INTERVAL_BYTES, DEFAULT_SYNC_INTERVAL_BYTES);
+ String compressionCodec = context.getString(COMPRESSION_CODEC, DEFAULT_COMPRESSION_CODEC);
+
+ writer = new ReflectDatumWriter<T>(getSchema());
+ dataFileWriter = new DataFileWriter<T>(writer);
+
+ dataFileWriter.setSyncInterval(syncIntervalBytes);
+
+ try {
+ CodecFactory codecFactory = CodecFactory.fromString(compressionCodec);
+ dataFileWriter.setCodec(codecFactory);
+ } catch (AvroRuntimeException e) {
+ logger.warn(
+ "Unable to instantiate avro codec with name (" + compressionCodec
+ + "). Compression disabled. Exception follows.",
+ e);
+ }
}
- }
-
- @Override
- public void afterCreate() throws IOException {
- // write the AVRO container format header
- dataFileWriter.create(getSchema(), getOutputStream());
- }
-
- @Override
- public void afterReopen() throws IOException {
- // impossible to initialize DataFileWriter without writing the schema?
- throw new UnsupportedOperationException("Avro API doesn't support append");
- }
-
- @Override
- public void write(Event event) throws IOException {
- T destType = convert(event);
- dataFileWriter.append(destType);
- }
-
- @Override
- public void flush() throws IOException {
- dataFileWriter.flush();
- }
-
-
- @Override
- public void beforeClose() throws IOException {
- // no-op
- }
-
- @Override
- public boolean supportsReopen() {
- return false;
- }
+ @Override
+ public void afterCreate() throws IOException {
+ // write the AVRO container format header
+ dataFileWriter.create(getSchema(), getOutputStream());
+ }
+
+ @Override
+ public void afterReopen() throws IOException {
+ // impossible to initialize DataFileWriter without writing the schema?
+ throw new UnsupportedOperationException("Avro API doesn't support append");
+ }
+
+ @Override
+ public void write(Event event) throws IOException {
+ T destType = convert(event);
+ dataFileWriter.append(destType);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ dataFileWriter.flush();
+ }
+
+ @Override
+ public void beforeClose() throws IOException {
+ // no-op
+ }
+
+ @Override
+ public boolean supportsReopen() {
+ return false;
+ }
}
diff --git a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AvroEventDeserializer.java b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AvroEventDeserializer.java
index dce1e49..8b5735b 100644
--- a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AvroEventDeserializer.java
+++ b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AvroEventDeserializer.java
@@ -1,13 +1,12 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -19,6 +18,11 @@ package org.apache.flume.rpc.avro.serialization;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
+import java.util.List;
+import java.util.Locale;
import org.apache.avro.Schema;
import org.apache.avro.SchemaNormalization;
import org.apache.avro.file.DataFileConstants;
@@ -41,12 +45,6 @@ import org.apache.flume.serialization.ResettableInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-import java.util.List;
-import java.util.Locale;
-
/**
* A deserializer that parses Avro container files, generating one Flume event
* per record in the Avro file, and storing binary avro-encoded records in
@@ -54,169 +52,162 @@ import java.util.Locale;
*/
public class AvroEventDeserializer implements EventDeserializer {
- private static final Logger logger = LoggerFactory.getLogger(AvroEventDeserializer.class);
-
- private final AvroSchemaType schemaType;
- private final ResettableInputStream ris;
-
- private Schema schema;
- private byte[] schemaHash;
- private String schemaHashString;
- private DataFileReader<GenericRecord> fileReader;
- private GenericDatumWriter datumWriter;
- private GenericRecord record;
- private ByteArrayOutputStream out;
- private BinaryEncoder encoder;
-
- @VisibleForTesting
- public static enum AvroSchemaType {
- HASH,
- LITERAL;
- }
-
- public static final String CONFIG_SCHEMA_TYPE_KEY = "schemaType";
- public static final String AVRO_SCHEMA_HEADER_HASH
- = "flume.avro.schema.hash";
- public static final String AVRO_SCHEMA_HEADER_LITERAL
- = "flume.avro.schema.literal";
-
- private AvroEventDeserializer(Context context, ResettableInputStream ris) {
- this.ris = ris;
-
- schemaType = AvroSchemaType.valueOf(
- context.getString(CONFIG_SCHEMA_TYPE_KEY,
- AvroSchemaType.HASH.toString()).toUpperCase(Locale.ENGLISH));
- if (schemaType == AvroSchemaType.LITERAL) {
- logger.warn(CONFIG_SCHEMA_TYPE_KEY + " set to " +
- AvroSchemaType.LITERAL.toString() + ", so storing full Avro " +
- "schema in the header of each event, which may be inefficient. " +
- "Consider using the hash of the schema " +
- "instead of the literal schema.");
- }
- }
-
- private void initialize() throws IOException, NoSuchAlgorithmException {
- SeekableResettableInputBridge in = new SeekableResettableInputBridge(ris);
- long pos = in.tell();
- in.seek(0L);
- fileReader = new DataFileReader<GenericRecord>(in,
- new GenericDatumReader<GenericRecord>());
- fileReader.sync(pos);
-
- schema = fileReader.getSchema();
- datumWriter = new GenericDatumWriter(schema);
- out = new ByteArrayOutputStream();
- encoder = EncoderFactory.get().binaryEncoder(out, encoder);
-
- schemaHash = SchemaNormalization.parsingFingerprint("CRC-64-AVRO", schema);
- schemaHashString = Hex.encodeHexString(schemaHash);
- }
-
- @Override
- public Event readEvent() throws IOException {
- if (fileReader.hasNext()) {
- record = fileReader.next(record);
- out.reset();
- datumWriter.write(record, encoder);
- encoder.flush();
- // annotate header with 64-bit schema CRC hash in hex
- Event event = EventBuilder.withBody(out.toByteArray());
- if (schemaType == AvroSchemaType.HASH) {
- event.getHeaders().put(AVRO_SCHEMA_HEADER_HASH, schemaHashString);
- } else {
- event.getHeaders().put(AVRO_SCHEMA_HEADER_LITERAL, schema.toString());
- }
- return event;
- }
- return null;
- }
-
- @Override
- public List<Event> readEvents(int numEvents) throws IOException {
- List<Event> events = Lists.newArrayList();
- for (int i = 0; i < numEvents && fileReader.hasNext(); i++) {
- Event event = readEvent();
- if (event != null) {
- events.add(event);
- }
- }
- return events;
- }
-
- @Override
- public void mark() throws IOException {
- long pos = fileReader.previousSync() - DataFileConstants.SYNC_SIZE;
- if (pos < 0) pos = 0;
- ((RemoteMarkable) ris).markPosition(pos);
- }
+ private static final Logger logger = LoggerFactory.getLogger(AvroEventDeserializer.class);
- @Override
- public void reset() throws IOException {
- long pos = ((RemoteMarkable) ris).getMarkPosition();
- fileReader.sync(pos);
- }
+ private final AvroSchemaType schemaType;
+ private final ResettableInputStream ris;
- @Override
- public void close() throws IOException {
- ris.close();
- }
+ private Schema schema;
+ private byte[] schemaHash;
+ private String schemaHashString;
+ private DataFileReader<GenericRecord> fileReader;
+ private GenericDatumWriter datumWriter;
+ private GenericRecord record;
+ private ByteArrayOutputStream out;
+ private BinaryEncoder encoder;
- public static class Builder implements EventDeserializer.Builder {
+ @VisibleForTesting
+ public static enum AvroSchemaType {
+ HASH,
+ LITERAL;
+ }
- @Override
- public EventDeserializer build(Context context, ResettableInputStream in) {
- if (!(in instanceof RemoteMarkable)) {
- throw new IllegalArgumentException("Cannot use this deserializer " +
- "without a RemoteMarkable input stream");
- }
- AvroEventDeserializer deserializer
- = new AvroEventDeserializer(context, in);
- try {
- deserializer.initialize();
- } catch (Exception e) {
- throw new FlumeException("Cannot instantiate deserializer", e);
- }
- return deserializer;
+ public static final String CONFIG_SCHEMA_TYPE_KEY = "schemaType";
+ public static final String AVRO_SCHEMA_HEADER_HASH = "flume.avro.schema.hash";
+ public static final String AVRO_SCHEMA_HEADER_LITERAL = "flume.avro.schema.literal";
+
+ private AvroEventDeserializer(Context context, ResettableInputStream ris) {
+ this.ris = ris;
+
+ schemaType = AvroSchemaType.valueOf(context.getString(CONFIG_SCHEMA_TYPE_KEY, AvroSchemaType.HASH.toString())
+ .toUpperCase(Locale.ENGLISH));
+ if (schemaType == AvroSchemaType.LITERAL) {
+ logger.warn(CONFIG_SCHEMA_TYPE_KEY + " set to " + AvroSchemaType.LITERAL.toString()
+ + ", so storing full Avro " + "schema in the header of each event, which may be inefficient. "
+ + "Consider using the hash of the schema "
+ + "instead of the literal schema.");
+ }
}
- }
- private static class SeekableResettableInputBridge implements SeekableInput {
- ResettableInputStream ris;
+ private void initialize() throws IOException, NoSuchAlgorithmException {
+ SeekableResettableInputBridge in = new SeekableResettableInputBridge(ris);
+ long pos = in.tell();
+ in.seek(0L);
+ fileReader = new DataFileReader<GenericRecord>(in, new GenericDatumReader<GenericRecord>());
+ fileReader.sync(pos);
+
+ schema = fileReader.getSchema();
+ datumWriter = new GenericDatumWriter(schema);
+ out = new ByteArrayOutputStream();
+ encoder = EncoderFactory.get().binaryEncoder(out, encoder);
- public SeekableResettableInputBridge(ResettableInputStream ris) {
- this.ris = ris;
+ schemaHash = SchemaNormalization.parsingFingerprint("CRC-64-AVRO", schema);
+ schemaHashString = Hex.encodeHexString(schemaHash);
}
@Override
- public void seek(long p) throws IOException {
- ris.seek(p);
+ public Event readEvent() throws IOException {
+ if (fileReader.hasNext()) {
+ record = fileReader.next(record);
+ out.reset();
+ datumWriter.write(record, encoder);
+ encoder.flush();
+ // annotate header with 64-bit schema CRC hash in hex
+ Event event = EventBuilder.withBody(out.toByteArray());
+ if (schemaType == AvroSchemaType.HASH) {
+ event.getHeaders().put(AVRO_SCHEMA_HEADER_HASH, schemaHashString);
+ } else {
+ event.getHeaders().put(AVRO_SCHEMA_HEADER_LITERAL, schema.toString());
+ }
+ return event;
+ }
+ return null;
}
@Override
- public long tell() throws IOException {
- return ris.tell();
+ public List<Event> readEvents(int numEvents) throws IOException {
+ List<Event> events = Lists.newArrayList();
+ for (int i = 0; i < numEvents && fileReader.hasNext(); i++) {
+ Event event = readEvent();
+ if (event != null) {
+ events.add(event);
+ }
+ }
+ return events;
}
@Override
- public long length() throws IOException {
- if (ris instanceof LengthMeasurable) {
- return ((LengthMeasurable) ris).length();
- } else {
- // FIXME: Avro doesn't seem to complain about this,
- // but probably not a great idea...
- return Long.MAX_VALUE;
- }
+ public void mark() throws IOException {
+ long pos = fileReader.previousSync() - DataFileConstants.SYNC_SIZE;
+ if (pos < 0) pos = 0;
+ ((RemoteMarkable) ris).markPosition(pos);
}
@Override
- public int read(byte[] b, int off, int len) throws IOException {
- return ris.read(b, off, len);
+ public void reset() throws IOException {
+ long pos = ((RemoteMarkable) ris).getMarkPosition();
+ fileReader.sync(pos);
}
@Override
public void close() throws IOException {
- ris.close();
+ ris.close();
}
- }
+ public static class Builder implements EventDeserializer.Builder {
+
+ @Override
+ public EventDeserializer build(Context context, ResettableInputStream in) {
+ if (!(in instanceof RemoteMarkable)) {
+ throw new IllegalArgumentException(
+ "Cannot use this deserializer " + "without a RemoteMarkable input stream");
+ }
+ AvroEventDeserializer deserializer = new AvroEventDeserializer(context, in);
+ try {
+ deserializer.initialize();
+ } catch (Exception e) {
+ throw new FlumeException("Cannot instantiate deserializer", e);
+ }
+ return deserializer;
+ }
+ }
+
+ private static class SeekableResettableInputBridge implements SeekableInput {
+ ResettableInputStream ris;
+
+ public SeekableResettableInputBridge(ResettableInputStream ris) {
+ this.ris = ris;
+ }
+
+ @Override
+ public void seek(long p) throws IOException {
+ ris.seek(p);
+ }
+
+ @Override
+ public long tell() throws IOException {
+ return ris.tell();
+ }
+
+ @Override
+ public long length() throws IOException {
+ if (ris instanceof LengthMeasurable) {
+ return ((LengthMeasurable) ris).length();
+ } else {
+ // FIXME: Avro doesn't seem to complain about this,
+ // but probably not a great idea...
+ return Long.MAX_VALUE;
+ }
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ return ris.read(b, off, len);
+ }
+
+ @Override
+ public void close() throws IOException {
+ ris.close();
+ }
+ }
}
diff --git a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AvroEventSerializerConfigurationConstants.java b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AvroEventSerializerConfigurationConstants.java
index 5b8007a..50d0e50 100644
--- a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AvroEventSerializerConfigurationConstants.java
+++ b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/AvroEventSerializerConfigurationConstants.java
@@ -1,20 +1,18 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.rpc.avro.serialization;
@@ -22,22 +20,25 @@ import org.apache.avro.file.CodecFactory;
public class AvroEventSerializerConfigurationConstants {
- /**
- * Avro sync interval, in approximate bytes
- */
- public static final String SYNC_INTERVAL_BYTES = "syncIntervalBytes";
- public static final int DEFAULT_SYNC_INTERVAL_BYTES = 2048000; // 2MB
-
- /**
- * Avro compression codec. For supported codecs, see Avro's
- * {@link CodecFactory} class.
- */
- public static final String COMPRESSION_CODEC = "compressionCodec";
- public static final String DEFAULT_COMPRESSION_CODEC = "null"; // no codec
-
- /**
- * Avro static Schema URL
- */
- public static final String STATIC_SCHEMA_URL = "schemaURL";
- public static final String DEFAULT_STATIC_SCHEMA_URL = null;
+ /**
+ * Avro sync interval, in approximate bytes
+ */
+ public static final String SYNC_INTERVAL_BYTES = "syncIntervalBytes";
+
+ public static final int DEFAULT_SYNC_INTERVAL_BYTES = 2048000; // 2MB
+
+ /**
+ * Avro compression codec. For supported codecs, see Avro's
+ * {@link CodecFactory} class.
+ */
+ public static final String COMPRESSION_CODEC = "compressionCodec";
+
+ public static final String DEFAULT_COMPRESSION_CODEC = "null"; // no codec
+
+ /**
+ * Avro static Schema URL
+ */
+ public static final String STATIC_SCHEMA_URL = "schemaURL";
+
+ public static final String DEFAULT_STATIC_SCHEMA_URL = null;
}
diff --git a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/DurablePositionTracker.java b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/DurablePositionTracker.java
index 16ee388..a540ab0 100644
--- a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/DurablePositionTracker.java
+++ b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/DurablePositionTracker.java
@@ -1,13 +1,12 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,13 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flume.rpc.avro.serialization;
+import com.google.common.base.Preconditions;
import java.io.File;
import java.io.IOException;
-
-import com.google.common.base.Preconditions;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.io.DatumReader;
@@ -48,156 +45,145 @@ import org.apache.flume.tools.PlatformDetect;
@InterfaceStability.Evolving
public class DurablePositionTracker implements PositionTracker {
- private final File trackerFile;
- private final DataFileWriter<TransferStateFileMeta> writer;
- private final DataFileReader<TransferStateFileMeta> reader;
- private final TransferStateFileMeta metaCache;
-
- private String target;
-
- private boolean isOpen;
-
- /**
- * If the file exists at startup, then read it, roll it, and open a new one.
- * We go through this to avoid issues with partial reads at the end of the
- * file from a previous crash. If we append to a bad record,
- * our writes may never be visible.
- * @param trackerFile
- * @param target
- * @return DurablePositionTracker
- * @throws IOException
- */
- public static DurablePositionTracker getInstance(File trackerFile,
- String target) throws IOException {
-
- if (!trackerFile.exists()) {
- return new DurablePositionTracker(trackerFile, target);
+ private final File trackerFile;
+ private final DataFileWriter<TransferStateFileMeta> writer;
+ private final DataFileReader<TransferStateFileMeta> reader;
+ private final TransferStateFileMeta metaCache;
+
+ private String target;
+
+ private boolean isOpen;
+
+ /**
+ * If the file exists at startup, then read it, roll it, and open a new one.
+ * We go through this to avoid issues with partial reads at the end of the
+ * file from a previous crash. If we append to a bad record,
+ * our writes may never be visible.
+ * @param trackerFile
+ * @param target
+ * @return DurablePositionTracker
+ * @throws IOException
+ */
+ public static DurablePositionTracker getInstance(File trackerFile, String target) throws IOException {
+
+ if (!trackerFile.exists()) {
+ return new DurablePositionTracker(trackerFile, target);
+ }
+
+ // exists
+ DurablePositionTracker oldTracker = new DurablePositionTracker(trackerFile, target);
+ String existingTarget = oldTracker.getTarget();
+ long targetPosition = oldTracker.getPosition();
+ oldTracker.close();
+
+ File tmpMeta = File.createTempFile(trackerFile.getName(), ".tmp", trackerFile.getParentFile());
+ tmpMeta.delete();
+ DurablePositionTracker tmpTracker = new DurablePositionTracker(tmpMeta, existingTarget);
+ tmpTracker.storePosition(targetPosition);
+ tmpTracker.close();
+
+ // On windows, things get messy with renames...
+ // FIXME: This is not atomic. Consider implementing a recovery procedure
+ // so that if it does not exist at startup, check for a rolled version
+ // before creating a new file from scratch.
+ if (PlatformDetect.isWindows()) {
+ if (!trackerFile.delete()) {
+ throw new IOException("Unable to delete existing meta file " + trackerFile);
+ }
+ }
+
+ // rename tmp file to meta
+ if (!tmpMeta.renameTo(trackerFile)) {
+ throw new IOException("Unable to rename " + tmpMeta + " to " + trackerFile);
+ }
+
+ // return a new known-good version that is open for append
+ DurablePositionTracker newTracker = new DurablePositionTracker(trackerFile, existingTarget);
+ return newTracker;
}
- // exists
- DurablePositionTracker oldTracker =
- new DurablePositionTracker(trackerFile, target);
- String existingTarget = oldTracker.getTarget();
- long targetPosition = oldTracker.getPosition();
- oldTracker.close();
-
- File tmpMeta = File.createTempFile(trackerFile.getName(), ".tmp",
- trackerFile.getParentFile());
- tmpMeta.delete();
- DurablePositionTracker tmpTracker =
- new DurablePositionTracker(tmpMeta, existingTarget);
- tmpTracker.storePosition(targetPosition);
- tmpTracker.close();
-
- // On windows, things get messy with renames...
- // FIXME: This is not atomic. Consider implementing a recovery procedure
- // so that if it does not exist at startup, check for a rolled version
- // before creating a new file from scratch.
- if (PlatformDetect.isWindows()) {
- if (!trackerFile.delete()) {
- throw new IOException("Unable to delete existing meta file " +
- trackerFile);
- }
- }
+ /**
+ * If the file exists, read it and open it for append.
+ * @param trackerFile
+ * @param target
+ * @throws IOException
+ */
+ DurablePositionTracker(File trackerFile, String target) throws IOException {
- // rename tmp file to meta
- if (!tmpMeta.renameTo(trackerFile)) {
- throw new IOException("Unable to rename " + tmpMeta + " to " +
- trackerFile);
- }
+ Preconditions.checkNotNull(trackerFile, "trackerFile must not be null");
+ Preconditions.checkNotNull(target, "target must not be null");
- // return a new known-good version that is open for append
- DurablePositionTracker newTracker =
- new DurablePositionTracker(trackerFile, existingTarget);
- return newTracker;
- }
-
- /**
- * If the file exists, read it and open it for append.
- * @param trackerFile
- * @param target
- * @throws IOException
- */
- DurablePositionTracker(File trackerFile, String target)
- throws IOException {
-
- Preconditions.checkNotNull(trackerFile, "trackerFile must not be null");
- Preconditions.checkNotNull(target, "target must not be null");
-
- this.trackerFile = trackerFile;
- this.target = target;
-
- DatumWriter<TransferStateFileMeta> dout =
- new SpecificDatumWriter<TransferStateFileMeta>(
- TransferStateFileMeta.SCHEMA$);
-
- DatumReader<TransferStateFileMeta> din =
- new SpecificDatumReader<TransferStateFileMeta>(
- TransferStateFileMeta.SCHEMA$);
-
- writer = new DataFileWriter<TransferStateFileMeta>(dout);
-
- if (trackerFile.exists()) {
- // open it for append
- writer.appendTo(trackerFile);
-
- reader = new DataFileReader<TransferStateFileMeta>(trackerFile, din);
- this.target = reader.getMetaString("file");
- } else {
- // create the file
- this.target = target;
- writer.setMeta("file", target);
- writer.create(TransferStateFileMeta.SCHEMA$, trackerFile);
- reader = new DataFileReader<TransferStateFileMeta>(trackerFile, din);
- }
+ this.trackerFile = trackerFile;
+ this.target = target;
+
+ DatumWriter<TransferStateFileMeta> dout =
+ new SpecificDatumWriter<TransferStateFileMeta>(TransferStateFileMeta.SCHEMA$);
+
+ DatumReader<TransferStateFileMeta> din =
+ new SpecificDatumReader<TransferStateFileMeta>(TransferStateFileMeta.SCHEMA$);
- target = getTarget();
+ writer = new DataFileWriter<TransferStateFileMeta>(dout);
- // initialize @ line = 0;
- metaCache = TransferStateFileMeta.newBuilder().setOffset(0L).build();
+ if (trackerFile.exists()) {
+ // open it for append
+ writer.appendTo(trackerFile);
- initReader();
+ reader = new DataFileReader<TransferStateFileMeta>(trackerFile, din);
+ this.target = reader.getMetaString("file");
+ } else {
+ // create the file
+ this.target = target;
+ writer.setMeta("file", target);
+ writer.create(TransferStateFileMeta.SCHEMA$, trackerFile);
+ reader = new DataFileReader<TransferStateFileMeta>(trackerFile, din);
+ }
- isOpen = true;
- }
+ target = getTarget();
- /**
- * Read the last record in the file.
- */
- private void initReader() throws IOException {
- long syncPos = trackerFile.length() - 256L;
- if (syncPos < 0) syncPos = 0L;
- reader.sync(syncPos);
- while (reader.hasNext()) {
- reader.next(metaCache);
+ // initialize @ line = 0;
+ metaCache = TransferStateFileMeta.newBuilder().setOffset(0L).build();
+
+ initReader();
+
+ isOpen = true;
+ }
+
+ /**
+ * Read the last record in the file.
+ */
+ private void initReader() throws IOException {
+ long syncPos = trackerFile.length() - 256L;
+ if (syncPos < 0) syncPos = 0L;
+ reader.sync(syncPos);
+ while (reader.hasNext()) {
+ reader.next(metaCache);
+ }
+ }
+
+ @Override
+ public synchronized void storePosition(long position) throws IOException {
+ metaCache.setOffset(position);
+ writer.append(metaCache);
+ writer.sync();
+ writer.flush();
}
- }
-
- @Override
- public synchronized void storePosition(long position) throws IOException {
- metaCache.setOffset(position);
- writer.append(metaCache);
- writer.sync();
- writer.flush();
- }
-
- @Override
- public synchronized long getPosition() {
- return metaCache.getOffset();
- }
-
- @Override
- public String getTarget() {
- return target;
- }
-
- @Override
- public void close() throws IOException {
- if (isOpen) {
- writer.close();
- reader.close();
- isOpen = false;
+
+ @Override
+ public synchronized long getPosition() {
+ return metaCache.getOffset();
}
- }
-}
\ No newline at end of file
+ @Override
+ public String getTarget() {
+ return target;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (isOpen) {
+ writer.close();
+ reader.close();
+ isOpen = false;
+ }
+ }
+}
diff --git a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/FlumeEventAvroEventSerializer.java b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/FlumeEventAvroEventSerializer.java
index 599ee30..1173ffd 100644
--- a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/FlumeEventAvroEventSerializer.java
+++ b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/FlumeEventAvroEventSerializer.java
@@ -1,20 +1,18 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.rpc.avro.serialization;
@@ -26,46 +24,44 @@ import org.apache.flume.serialization.EventSerializer;
public class FlumeEventAvroEventSerializer extends AbstractAvroEventSerializer<Event> {
- private static final Schema SCHEMA = new Schema.Parser().parse(
- "{ \"type\":\"record\", \"name\": \"Event\", \"fields\": [" +
- " {\"name\": \"headers\", \"type\": { \"type\": \"map\", \"values\": \"string\" } }, " +
- " {\"name\": \"body\", \"type\": \"bytes\" } ] }");
+ private static final Schema SCHEMA = new Schema.Parser()
+ .parse("{ \"type\":\"record\", \"name\": \"Event\", \"fields\": ["
+ + " {\"name\": \"headers\", \"type\": { \"type\": \"map\", \"values\": \"string\" } }, "
+ + " {\"name\": \"body\", \"type\": \"bytes\" } ] }");
- private final OutputStream out;
+ private final OutputStream out;
- private FlumeEventAvroEventSerializer(OutputStream out) {
- this.out = out;
- }
-
- @Override
- protected Schema getSchema() {
- return SCHEMA;
- }
-
- @Override
- protected OutputStream getOutputStream() {
- return out;
- }
+ private FlumeEventAvroEventSerializer(OutputStream out) {
+ this.out = out;
+ }
- /**
- * A no-op for this simple, special-case implementation
- * @param event
- * @return
- */
- @Override
- protected Event convert(Event event) {
- return event;
- }
+ @Override
+ protected Schema getSchema() {
+ return SCHEMA;
+ }
- public static class Builder implements EventSerializer.Builder {
+ @Override
+ protected OutputStream getOutputStream() {
+ return out;
+ }
+ /**
+ * A no-op for this simple, special-case implementation
+ * @param event
+ * @return
+ */
@Override
- public EventSerializer build(Context context, OutputStream out) {
- FlumeEventAvroEventSerializer writer = new FlumeEventAvroEventSerializer(out);
- writer.configure(context);
- return writer;
+ protected Event convert(Event event) {
+ return event;
}
- }
+ public static class Builder implements EventSerializer.Builder {
+ @Override
+ public EventSerializer build(Context context, OutputStream out) {
+ FlumeEventAvroEventSerializer writer = new FlumeEventAvroEventSerializer(out);
+ writer.configure(context);
+ return writer;
+ }
+ }
}
diff --git a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/ReliableSpoolingFileEventReader.java b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/ReliableSpoolingFileEventReader.java
index 26e0a46..5662893 100644
--- a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/ReliableSpoolingFileEventReader.java
+++ b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/serialization/ReliableSpoolingFileEventReader.java
@@ -1,48 +1,25 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
-
package org.apache.flume.rpc.avro.serialization;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.FlumeException;
-import org.apache.flume.annotations.InterfaceAudience;
-import org.apache.flume.annotations.InterfaceStability;
-import org.apache.flume.event.EventBuilder;
-import org.apache.flume.event.ReliableEventReader;
-import org.apache.flume.instrumentation.SourceCounter;
-import org.apache.flume.serialization.DecodeErrorPolicy;
-import org.apache.flume.serialization.EventDeserializer;
-import org.apache.flume.serialization.EventDeserializerFactory;
-import org.apache.flume.serialization.PositionTracker;
-import org.apache.flume.serialization.ResettableFileInputStream;
-import org.apache.flume.serialization.ResettableInputStream;
-import org.apache.flume.source.SpoolDirectorySourceConfigurationConstants;
-import org.apache.flume.source.SpoolDirectorySourceConfigurationConstants.ConsumeOrder;
-import org.apache.flume.tools.PlatformDetect;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -64,6 +41,25 @@ import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.regex.Pattern;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.FlumeException;
+import org.apache.flume.annotations.InterfaceAudience;
+import org.apache.flume.annotations.InterfaceStability;
+import org.apache.flume.event.EventBuilder;
+import org.apache.flume.event.ReliableEventReader;
+import org.apache.flume.instrumentation.SourceCounter;
+import org.apache.flume.serialization.DecodeErrorPolicy;
+import org.apache.flume.serialization.EventDeserializer;
+import org.apache.flume.serialization.EventDeserializerFactory;
+import org.apache.flume.serialization.PositionTracker;
+import org.apache.flume.serialization.ResettableFileInputStream;
+import org.apache.flume.serialization.ResettableInputStream;
+import org.apache.flume.source.SpoolDirectorySourceConfigurationConstants;
+import org.apache.flume.source.SpoolDirectorySourceConfigurationConstants.ConsumeOrder;
+import org.apache.flume.tools.PlatformDetect;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* <p>A {@link ReliableEventReader} which reads log data from files stored
@@ -91,821 +87,817 @@ import java.util.regex.Pattern;
@InterfaceStability.Evolving
public class ReliableSpoolingFileEventReader implements ReliableEventReader {
- private static final Logger logger = LoggerFactory
- .getLogger(ReliableSpoolingFileEventReader.class);
-
- static final String metaFileName = ".flumespool-main.meta";
- private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS");
- private final File spoolDirectory;
- private final Path spoolDirPath;
- private final String completedSuffix;
- private final String deserializerType;
- private final Context deserializerContext;
- private final Pattern includePattern;
- private final Pattern ignorePattern;
- private final File metaFile;
- private File trackerDirectory;
- private final boolean annotateFileName;
- private final boolean annotateBaseName;
- private final String fileNameHeader;
- private final String baseNameHeader;
- private final String deletePolicy;
- private final TrackingPolicy trackingPolicy;
- private final Charset inputCharset;
- private final DecodeErrorPolicy decodeErrorPolicy;
- private final ConsumeOrder consumeOrder;
- private final boolean recursiveDirectorySearch;
- private final SourceCounter sourceCounter;
-
- private Optional<FileInfo> currentFile = Optional.absent();
- /** Always contains the last file from which lines have been read. */
- private Optional<FileInfo> lastFileRead = Optional.absent();
- private boolean committed = true;
- private boolean firstTimeRead = true;
-
- /** Instance var to Cache directory listing */
- private Iterator<File> candidateFileIter = null;
- private int listFilesCount = 0;
-
- private String trackerDirectoryAbsolutePath;
-
- /**
- * Create a ReliableSpoolingFileEventReader to watch the given directory.
- */
- private ReliableSpoolingFileEventReader(File spoolDirectory,
- String completedSuffix, String includePattern, String ignorePattern, String trackerDirPath,
- boolean annotateFileName, String fileNameHeader,
- boolean annotateBaseName, String baseNameHeader,
- String deserializerType, Context deserializerContext,
- String deletePolicy, String trackingPolicy, String inputCharset,
- DecodeErrorPolicy decodeErrorPolicy,
- ConsumeOrder consumeOrder, boolean recursiveDirectorySearch,
- SourceCounter sourceCounter) throws IOException {
-
- // Sanity checks
- Preconditions.checkNotNull(spoolDirectory);
- Preconditions.checkNotNull(completedSuffix);
- Preconditions.checkNotNull(includePattern);
- Preconditions.checkNotNull(ignorePattern);
- Preconditions.checkNotNull(trackerDirPath);
- Preconditions.checkNotNull(deserializerType);
- Preconditions.checkNotNull(deserializerContext);
- Preconditions.checkNotNull(deletePolicy);
- Preconditions.checkNotNull(trackingPolicy);
- Preconditions.checkNotNull(inputCharset);
- Preconditions.checkNotNull(sourceCounter);
-
- // validate delete policy
- if (!deletePolicy.equalsIgnoreCase(DeletePolicy.NEVER.name()) &&
- !deletePolicy.equalsIgnoreCase(DeletePolicy.IMMEDIATE.name())) {
- throw new IllegalArgumentException("Delete policies other than " +
- "NEVER and IMMEDIATE are not yet supported");
- }
+ private static final Logger logger = LoggerFactory.getLogger(ReliableSpoolingFileEventReader.class);
+
+ static final String metaFileName = ".flumespool-main.meta";
+ private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS");
+ private final File spoolDirectory;
+ private final Path spoolDirPath;
+ private final String completedSuffix;
+ private final String deserializerType;
+ private final Context deserializerContext;
+ private final Pattern includePattern;
+ private final Pattern ignorePattern;
+ private final File metaFile;
+ private File trackerDirectory;
+ private final boolean annotateFileName;
+ private final boolean annotateBaseName;
+ private final String fileNameHeader;
+ private final String baseNameHeader;
+ private final String deletePolicy;
+ private final TrackingPolicy trackingPolicy;
+ private final Charset inputCharset;
+ private final DecodeErrorPolicy decodeErrorPolicy;
+ private final ConsumeOrder consumeOrder;
+ private final boolean recursiveDirectorySearch;
+ private final SourceCounter sourceCounter;
+
+ private Optional<FileInfo> currentFile = Optional.absent();
+ /** Always contains the last file from which lines have been read. */
+ private Optional<FileInfo> lastFileRead = Optional.absent();
+
+ private boolean committed = true;
+ private boolean firstTimeRead = true;
+
+ /** Instance var to Cache directory listing */
+ private Iterator<File> candidateFileIter = null;
+
+ private int listFilesCount = 0;
+
+ private String trackerDirectoryAbsolutePath;
+
+ /**
+ * Create a ReliableSpoolingFileEventReader to watch the given directory.
+ */
+ private ReliableSpoolingFileEventReader(
+ File spoolDirectory,
+ String completedSuffix,
+ String includePattern,
+ String ignorePattern,
+ String trackerDirPath,
+ boolean annotateFileName,
+ String fileNameHeader,
+ boolean annotateBaseName,
+ String baseNameHeader,
+ String deserializerType,
+ Context deserializerContext,
+ String deletePolicy,
+ String trackingPolicy,
+ String inputCharset,
+ DecodeErrorPolicy decodeErrorPolicy,
+ ConsumeOrder consumeOrder,
+ boolean recursiveDirectorySearch,
+ SourceCounter sourceCounter)
+ throws IOException {
- // validate tracking policy
- if (!trackingPolicy.equalsIgnoreCase(TrackingPolicy.RENAME.name()) &&
- !trackingPolicy.equalsIgnoreCase(TrackingPolicy.TRACKER_DIR.name())) {
- throw new IllegalArgumentException("Tracking policies other than " +
- "RENAME and TRACKER_DIR are not supported");
- }
+ // Sanity checks
+ Preconditions.checkNotNull(spoolDirectory);
+ Preconditions.checkNotNull(completedSuffix);
+ Preconditions.checkNotNull(includePattern);
+ Preconditions.checkNotNull(ignorePattern);
+ Preconditions.checkNotNull(trackerDirPath);
+ Preconditions.checkNotNull(deserializerType);
+ Preconditions.checkNotNull(deserializerContext);
+ Preconditions.checkNotNull(deletePolicy);
+ Preconditions.checkNotNull(trackingPolicy);
+ Preconditions.checkNotNull(inputCharset);
+ Preconditions.checkNotNull(sourceCounter);
+
+ // validate delete policy
+ if (!deletePolicy.equalsIgnoreCase(DeletePolicy.NEVER.name())
+ && !deletePolicy.equalsIgnoreCase(DeletePolicy.IMMEDIATE.name())) {
+ throw new IllegalArgumentException(
+ "Delete policies other than " + "NEVER and IMMEDIATE are not yet supported");
+ }
- if (logger.isDebugEnabled()) {
- logger.debug("Initializing {} with directory={}, metaDir={}, " +
- "deserializer={}",
- new Object[] {
- ReliableSpoolingFileEventReader.class.getSimpleName(),
- spoolDirectory, trackerDirPath, deserializerType
- });
- }
+ // validate tracking policy
+ if (!trackingPolicy.equalsIgnoreCase(TrackingPolicy.RENAME.name())
+ && !trackingPolicy.equalsIgnoreCase(TrackingPolicy.TRACKER_DIR.name())) {
+ throw new IllegalArgumentException(
+ "Tracking policies other than " + "RENAME and TRACKER_DIR are not supported");
+ }
- // Verify directory exists and is readable/writable
- Preconditions.checkState(spoolDirectory.exists(),
- "Directory does not exist: " + spoolDirectory.getAbsolutePath());
- Preconditions.checkState(spoolDirectory.isDirectory(),
- "Path is not a directory: " + spoolDirectory.getAbsolutePath());
-
- if(trackingPolicy.equalsIgnoreCase(TrackingPolicy.RENAME.name())) {
- // Do a canary test to make sure we have access to spooling directory
- try {
- File canary = File.createTempFile("flume-spooldir-perm-check-", ".canary",
- spoolDirectory);
- Files.write(canary.toPath(), "testing flume file permissions\n".getBytes());
- List<String> lines = Files.readAllLines(canary.toPath(), Charsets.UTF_8);
- Preconditions.checkState(!lines.isEmpty(), "Empty canary file %s", canary);
- if (!canary.delete()) {
- throw new IOException("Unable to delete canary file " + canary);
- }
- logger.debug("Successfully created and deleted canary file: {}", canary);
- } catch (IOException e) {
- throw new FlumeException("Unable to read and modify files" +
- " in the spooling directory: " + spoolDirectory, e);
- }
- }
+ if (logger.isDebugEnabled()) {
+ logger.debug("Initializing {} with directory={}, metaDir={}, " + "deserializer={}", new Object[] {
+ ReliableSpoolingFileEventReader.class.getSimpleName(), spoolDirectory, trackerDirPath, deserializerType
+ });
+ }
- this.spoolDirectory = spoolDirectory;
- this.completedSuffix = completedSuffix;
- this.deserializerType = deserializerType;
- this.deserializerContext = deserializerContext;
- this.annotateFileName = annotateFileName;
- this.fileNameHeader = fileNameHeader;
- this.annotateBaseName = annotateBaseName;
- this.baseNameHeader = baseNameHeader;
- this.includePattern = Pattern.compile(includePattern);
- this.ignorePattern = Pattern.compile(ignorePattern);
- this.deletePolicy = deletePolicy;
- this.trackingPolicy = TrackingPolicy.valueOf(trackingPolicy.toUpperCase());
- this.inputCharset = Charset.forName(inputCharset);
- this.decodeErrorPolicy = Preconditions.checkNotNull(decodeErrorPolicy);
- this.consumeOrder = Preconditions.checkNotNull(consumeOrder);
- this.recursiveDirectorySearch = recursiveDirectorySearch;
- this.sourceCounter = sourceCounter;
-
- trackerDirectory = new File(trackerDirPath);
-
- // if relative path, treat as relative to spool directory
- if (!trackerDirectory.isAbsolute()) {
- trackerDirectory = new File(spoolDirectory, trackerDirPath);
- }
+ // Verify directory exists and is readable/writable
+ Preconditions.checkState(
+ spoolDirectory.exists(), "Directory does not exist: " + spoolDirectory.getAbsolutePath());
+ Preconditions.checkState(
+ spoolDirectory.isDirectory(), "Path is not a directory: " + spoolDirectory.getAbsolutePath());
+
+ if (trackingPolicy.equalsIgnoreCase(TrackingPolicy.RENAME.name())) {
+ // Do a canary test to make sure we have access to spooling directory
+ try {
+ File canary = File.createTempFile("flume-spooldir-perm-check-", ".canary", spoolDirectory);
+ Files.write(canary.toPath(), "testing flume file permissions\n".getBytes());
+ List<String> lines = Files.readAllLines(canary.toPath(), Charsets.UTF_8);
+ Preconditions.checkState(!lines.isEmpty(), "Empty canary file %s", canary);
+ if (!canary.delete()) {
+ throw new IOException("Unable to delete canary file " + canary);
+ }
+ logger.debug("Successfully created and deleted canary file: {}", canary);
+ } catch (IOException e) {
+ throw new FlumeException(
+ "Unable to read and modify files" + " in the spooling directory: " + spoolDirectory, e);
+ }
+ }
- // ensure that meta directory exists
- if (!trackerDirectory.exists()) {
- if (!trackerDirectory.mkdir()) {
- throw new IOException("Unable to mkdir nonexistent meta directory " +
- trackerDirectory);
- }
- }
+ this.spoolDirectory = spoolDirectory;
+ this.completedSuffix = completedSuffix;
+ this.deserializerType = deserializerType;
+ this.deserializerContext = deserializerContext;
+ this.annotateFileName = annotateFileName;
+ this.fileNameHeader = fileNameHeader;
+ this.annotateBaseName = annotateBaseName;
+ this.baseNameHeader = baseNameHeader;
+ this.includePattern = Pattern.compile(includePattern);
+ this.ignorePattern = Pattern.compile(ignorePattern);
+ this.deletePolicy = deletePolicy;
+ this.trackingPolicy = TrackingPolicy.valueOf(trackingPolicy.toUpperCase());
+ this.inputCharset = Charset.forName(inputCharset);
+ this.decodeErrorPolicy = Preconditions.checkNotNull(decodeErrorPolicy);
+ this.consumeOrder = Preconditions.checkNotNull(consumeOrder);
+ this.recursiveDirectorySearch = recursiveDirectorySearch;
+ this.sourceCounter = sourceCounter;
+
+ trackerDirectory = new File(trackerDirPath);
+
+ // if relative path, treat as relative to spool directory
+ if (!trackerDirectory.isAbsolute()) {
+ trackerDirectory = new File(spoolDirectory, trackerDirPath);
+ }
- // ensure that the meta directory is a directory
- if (!trackerDirectory.isDirectory()) {
- throw new IOException("Specified meta directory is not a directory" +
- trackerDirectory);
- }
+ // ensure that meta directory exists
+ if (!trackerDirectory.exists()) {
+ if (!trackerDirectory.mkdir()) {
+ throw new IOException("Unable to mkdir nonexistent meta directory " + trackerDirectory);
+ }
+ }
+
+ // ensure that the meta directory is a directory
+ if (!trackerDirectory.isDirectory()) {
+ throw new IOException("Specified meta directory is not a directory" + trackerDirectory);
+ }
- this.metaFile = new File(trackerDirectory, metaFileName);
+ this.metaFile = new File(trackerDirectory, metaFileName);
- if (metaFile.exists() && metaFile.length() == 0) {
- deleteMetaFile();
- }
+ if (metaFile.exists() && metaFile.length() == 0) {
+ deleteMetaFile();
+ }
- spoolDirPath = Paths.get(spoolDirectory.getAbsolutePath());
- trackerDirectoryAbsolutePath = trackerDirectory.getAbsolutePath();
-
- }
-
- /**
- * Recursively gather candidate files
- *
- * @param directory the directory to gather files from
- * @return list of files within the passed in directory
- */
- private List<File> getCandidateFiles(final Path directory) {
- Preconditions.checkNotNull(directory);
- final List<File> candidateFiles = new ArrayList<>();
- try {
- final Set<Path> trackerDirCompletedFiles = getTrackerDirCompletedFiles();
- Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
- @Override
- public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
- throws IOException {
- if (directory.equals(dir)) { // The top directory should always be listed
- return FileVisitResult.CONTINUE;
- }
- String directoryName = dir.getFileName().toString();
- if (!recursiveDirectorySearch ||
- directoryName.startsWith(".") ||
- ignorePattern.matcher(directoryName).matches()) {
- return FileVisitResult.SKIP_SUBTREE;
- }
- return FileVisitResult.CONTINUE;
- }
-
- @Override
- public FileVisitResult visitFile(Path candidate, BasicFileAttributes attrs)
- throws IOException {
- String fileName = candidate.getFileName().toString();
- if (!fileName.endsWith(completedSuffix) &&
- !isFileInTrackerDir(trackerDirCompletedFiles, candidate) &&
- !fileName.startsWith(".") &&
- includePattern.matcher(fileName).matches() &&
- !ignorePattern.matcher(fileName).matches()) {
- candidateFiles.add(candidate.toFile());
- }
-
- return FileVisitResult.CONTINUE;
- }
- });
- } catch (IOException e) {
- logger.error("I/O exception occurred while listing directories. " +
- "Files already matched will be returned. " + directory, e);
- sourceCounter.incrementGenericProcessingFail();
+ spoolDirPath = Paths.get(spoolDirectory.getAbsolutePath());
+ trackerDirectoryAbsolutePath = trackerDirectory.getAbsolutePath();
+ }
+
+ /**
+ * Recursively gather candidate files
+ *
+ * @param directory the directory to gather files from
+ * @return list of files within the passed in directory
+ */
+ private List<File> getCandidateFiles(final Path directory) {
+ Preconditions.checkNotNull(directory);
+ final List<File> candidateFiles = new ArrayList<>();
+ try {
+ final Set<Path> trackerDirCompletedFiles = getTrackerDirCompletedFiles();
+ Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+ if (directory.equals(dir)) { // The top directory should always be listed
+ return FileVisitResult.CONTINUE;
+ }
+ String directoryName = dir.getFileName().toString();
+ if (!recursiveDirectorySearch
+ || directoryName.startsWith(".")
+ || ignorePattern.matcher(directoryName).matches()) {
+ return FileVisitResult.SKIP_SUBTREE;
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path candidate, BasicFileAttributes attrs) throws IOException {
+ String fileName = candidate.getFileName().toString();
+ if (!fileName.endsWith(completedSuffix)
+ && !isFileInTrackerDir(trackerDirCompletedFiles, candidate)
+ && !fileName.startsWith(".")
+ && includePattern.matcher(fileName).matches()
+ && !ignorePattern.matcher(fileName).matches()) {
+ candidateFiles.add(candidate.toFile());
+ }
+
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ } catch (IOException e) {
+ logger.error(
+ "I/O exception occurred while listing directories. " + "Files already matched will be returned. "
+ + directory,
+ e);
+ sourceCounter.incrementGenericProcessingFail();
+ }
+
+ return candidateFiles;
}
- return candidateFiles;
- }
+ private Set<Path> getTrackerDirCompletedFiles() throws IOException {
+ final Set<Path> completedFiles = new HashSet<>();
+ if (TrackingPolicy.TRACKER_DIR != trackingPolicy) {
+ return completedFiles;
+ }
- private Set<Path> getTrackerDirCompletedFiles() throws IOException {
- final Set<Path> completedFiles = new HashSet<>();
- if (TrackingPolicy.TRACKER_DIR != trackingPolicy) {
- return completedFiles;
+ Path trackerDirPath = trackerDirectory.toPath();
+ Files.walkFileTree(trackerDirPath, new SimpleFileVisitor<Path>() {
+
+ @Override
+ public FileVisitResult visitFile(Path candidate, BasicFileAttributes attrs) throws IOException {
+ String fileName = candidate.getFileName().toString();
+ if (fileName.endsWith(completedSuffix)) {
+ completedFiles.add(candidate.toAbsolutePath());
+ }
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ return completedFiles;
}
- Path trackerDirPath = trackerDirectory.toPath();
- Files.walkFileTree(trackerDirPath, new SimpleFileVisitor<Path>() {
-
- @Override
- public FileVisitResult visitFile(Path candidate, BasicFileAttributes attrs)
- throws IOException {
- String fileName = candidate.getFileName().toString();
- if (fileName.endsWith(completedSuffix)) {
- completedFiles.add(candidate.toAbsolutePath());
- }
- return FileVisitResult.CONTINUE;
- }
- });
- return completedFiles;
- }
-
- private boolean isFileInTrackerDir(Set<Path> completedFiles, Path path) {
- Path relPath = getRelPathToSpoolDir(path);
- Path trackerPath = Paths.get(trackerDirectoryAbsolutePath, relPath.toString() + completedSuffix);
- return completedFiles.contains(trackerPath);
- }
-
- private Path getRelPathToSpoolDir(Path path) {
- return spoolDirPath.relativize(path.toAbsolutePath());
- }
-
- @VisibleForTesting
- int getListFilesCount() {
- return listFilesCount;
- }
-
- /**
- * Return the filename which generated the data from the last successful
- * {@link #readEvents(int)} call. Returns null if called before any file
- * contents are read.
- */
- public String getLastFileRead() {
- if (!lastFileRead.isPresent()) {
- return null;
+ private boolean isFileInTrackerDir(Set<Path> completedFiles, Path path) {
+ Path relPath = getRelPathToSpoolDir(path);
+ Path trackerPath = Paths.get(trackerDirectoryAbsolutePath, relPath.toString() + completedSuffix);
+ return completedFiles.contains(trackerPath);
}
- return lastFileRead.get().getFile().getAbsolutePath();
- }
-
- /**
- * Return the filename, lastModified, and size which generated the data from the last successful
- * {@link #readEvents(int)} call. Returns null if called before any file
- * contents are read.
- */
- public String getLastFileReadInfo() {
- if (!lastFileRead.isPresent()) {
- return null;
+
+ private Path getRelPathToSpoolDir(Path path) {
+ return spoolDirPath.relativize(path.toAbsolutePath());
}
- FileInfo fileInfo = lastFileRead.get();
- StringBuilder sb = new StringBuilder(fileInfo.getFile().getName());
- sb.append(" lastModified: ").append(FORMATTER.format(Instant.ofEpochMilli(fileInfo.lastModified)
- .atZone(ZoneId.systemDefault()).toLocalDateTime()));
- sb.append(" size: ").append(fileInfo.length);
- return sb.toString();
- }
-
- // public interface
- public Event readEvent() throws IOException {
- List<Event> events = readEvents(1);
- if (!events.isEmpty()) {
- return events.get(0);
- } else {
- return null;
+
+ @VisibleForTesting
+ int getListFilesCount() {
+ return listFilesCount;
}
- }
-
- public List<Event> readEvents(int numEvents) throws IOException {
- if (!committed) {
- if (!currentFile.isPresent()) {
- throw new IllegalStateException("File should not roll when " +
- "commit is outstanding.");
- }
- logger.info("Last read was never committed - resetting mark position.");
- currentFile.get().getDeserializer().reset();
- } else {
- // Check if new files have arrived since last call
- if (!currentFile.isPresent()) {
- currentFile = getNextFile();
- }
- // Return empty list if no new files
- if (!currentFile.isPresent()) {
- return Collections.emptyList();
- }
+
+ /**
+ * Return the filename which generated the data from the last successful
+ * {@link #readEvents(int)} call. Returns null if called before any file
+ * contents are read.
+ */
+ public String getLastFileRead() {
+ if (!lastFileRead.isPresent()) {
+ return null;
+ }
+ return lastFileRead.get().getFile().getAbsolutePath();
}
- List<Event> events = readDeserializerEvents(numEvents);
-
- /* It's possible that the last read took us just up to a file boundary.
- * If so, try to roll to the next file, if there is one.
- * Loop until events is not empty or there is no next file in case of 0 byte files */
- while (events.isEmpty()) {
- logger.info("Last read took us just up to a file boundary. " +
- "Rolling to the next file, if there is one.");
- retireCurrentFile();
- currentFile = getNextFile();
- if (!currentFile.isPresent()) {
- return Collections.emptyList();
- }
- events = readDeserializerEvents(numEvents);
+ /**
+ * Return the filename, lastModified, and size which generated the data from the last successful
+ * {@link #readEvents(int)} call. Returns null if called before any file
+ * contents are read.
+ */
+ public String getLastFileReadInfo() {
+ if (!lastFileRead.isPresent()) {
+ return null;
+ }
+ FileInfo fileInfo = lastFileRead.get();
+ StringBuilder sb = new StringBuilder(fileInfo.getFile().getName());
+ sb.append(" lastModified: ")
+ .append(FORMATTER.format(Instant.ofEpochMilli(fileInfo.lastModified)
+ .atZone(ZoneId.systemDefault())
+ .toLocalDateTime()));
+ sb.append(" size: ").append(fileInfo.length);
+ return sb.toString();
+ }
+
+ // public interface
+ public Event readEvent() throws IOException {
+ List<Event> events = readEvents(1);
+ if (!events.isEmpty()) {
+ return events.get(0);
+ } else {
+ return null;
+ }
}
- fillHeader(events);
+ public List<Event> readEvents(int numEvents) throws IOException {
+ if (!committed) {
+ if (!currentFile.isPresent()) {
+ throw new IllegalStateException("File should not roll when " + "commit is outstanding.");
+ }
+ logger.info("Last read was never committed - resetting mark position.");
+ currentFile.get().getDeserializer().reset();
+ } else {
+ // Check if new files have arrived since last call
+ if (!currentFile.isPresent()) {
+ currentFile = getNextFile();
+ }
+ // Return empty list if no new files
+ if (!currentFile.isPresent()) {
+ return Collections.emptyList();
+ }
+ }
+
+ List<Event> events = readDeserializerEvents(numEvents);
+
+ /* It's possible that the last read took us just up to a file boundary.
+ * If so, try to roll to the next file, if there is one.
+ * Loop until events is not empty or there is no next file in case of 0 byte files */
+ while (events.isEmpty()) {
+ logger.info(
+ "Last read took us just up to a file boundary. " + "Rolling to the next file, if there is one.");
+ retireCurrentFile();
+ currentFile = getNextFile();
+ if (!currentFile.isPresent()) {
+ return Collections.emptyList();
+ }
+ events = readDeserializerEvents(numEvents);
+ }
- committed = false;
- lastFileRead = currentFile;
- return events;
- }
+ fillHeader(events);
- private List<Event> readDeserializerEvents(int numEvents) throws IOException {
- EventDeserializer des = currentFile.get().getDeserializer();
- List<Event> events = des.readEvents(numEvents);
- if (events.isEmpty() && firstTimeRead) {
- events.add(EventBuilder.withBody(new byte[0]));
+ committed = false;
+ lastFileRead = currentFile;
+ return events;
}
- firstTimeRead = false;
- return events;
- }
-
- private void fillHeader(List<Event> events) {
- if (annotateFileName) {
- String filename = currentFile.get().getFile().getAbsolutePath();
- for (Event event : events) {
- event.getHeaders().put(fileNameHeader, filename);
- }
+
+ private List<Event> readDeserializerEvents(int numEvents) throws IOException {
+ EventDeserializer des = currentFile.get().getDeserializer();
+ List<Event> events = des.readEvents(numEvents);
+ if (events.isEmpty() && firstTimeRead) {
+ events.add(EventBuilder.withBody(new byte[0]));
+ }
+ firstTimeRead = false;
+ return events;
}
- if (annotateBaseName) {
- String basename = currentFile.get().getFile().getName();
- for (Event event : events) {
- event.getHeaders().put(baseNameHeader, basename);
- }
+ private void fillHeader(List<Event> events) {
+ if (annotateFileName) {
+ String filename = currentFile.get().getFile().getAbsolutePath();
+ for (Event event : events) {
+ event.getHeaders().put(fileNameHeader, filename);
+ }
+ }
+
+ if (annotateBaseName) {
+ String basename = currentFile.get().getFile().getName();
+ for (Event event : events) {
+ event.getHeaders().put(baseNameHeader, basename);
+ }
+ }
}
- }
- @Override
- public void close() throws IOException {
- if (currentFile.isPresent()) {
- currentFile.get().getDeserializer().close();
- currentFile = Optional.absent();
+ @Override
+ public void close() throws IOException {
+ if (currentFile.isPresent()) {
+ currentFile.get().getDeserializer().close();
+ currentFile = Optional.absent();
+ }
}
- }
-
- /**
- * Commit the last lines which were read.
- */
- @Override
- public void commit() throws IOException {
- if (!committed && currentFile.isPresent()) {
- currentFile.get().getDeserializer().mark();
- committed = true;
+
+ /**
+ * Commit the last lines which were read.
+ */
+ @Override
+ public void commit() throws IOException {
+ if (!committed && currentFile.isPresent()) {
+ currentFile.get().getDeserializer().mark();
+ committed = true;
+ }
}
- }
-
- /**
- * Closes currentFile and attempt to rename it.
- * <p>
- * If these operations fail in a way that may cause duplicate log entries,
- * an error is logged but no exceptions are thrown. If these operations fail
- * in a way that indicates potential misuse of the spooling directory, a
- * FlumeException will be thrown.
- *
- * @throws FlumeException if files do not conform to spooling assumptions
- */
- private void retireCurrentFile() throws IOException {
- Preconditions.checkState(currentFile.isPresent());
-
- File fileToRoll = new File(currentFile.get().getFile().getAbsolutePath());
-
- currentFile.get().getDeserializer().close();
-
- // Verify that spooling assumptions hold
- if (fileToRoll.lastModified() != currentFile.get().getLastModified()) {
- String message = "File has been modified since being read: " + fileToRoll;
- throw new IllegalStateException(message);
+
+ /**
+ * Closes currentFile and attempt to rename it.
+ * <p>
+ * If these operations fail in a way that may cause duplicate log entries,
+ * an error is logged but no exceptions are thrown. If these operations fail
+ * in a way that indicates potential misuse of the spooling directory, a
+ * FlumeException will be thrown.
+ *
+ * @throws FlumeException if files do not conform to spooling assumptions
+ */
+ private void retireCurrentFile() throws IOException {
+ Preconditions.checkState(currentFile.isPresent());
+
+ File fileToRoll = new File(currentFile.get().getFile().getAbsolutePath());
+
+ currentFile.get().getDeserializer().close();
+
+ // Verify that spooling assumptions hold
+ if (fileToRoll.lastModified() != currentFile.get().getLastModified()) {
+ String message = "File has been modified since being read: " + fileToRoll;
+ throw new IllegalStateException(message);
+ }
+ if (fileToRoll.length() != currentFile.get().getLength()) {
+ String message = "File has changed size since being read: " + fileToRoll;
+ throw new IllegalStateException(message);
+ }
+
+ if (deletePolicy.equalsIgnoreCase(DeletePolicy.NEVER.name())) {
+ if (trackingPolicy == TrackingPolicy.RENAME) {
+ rollCurrentFile(fileToRoll);
+ } else {
+ rollCurrentFileInTrackerDir(fileToRoll);
+ }
+ } else if (deletePolicy.equalsIgnoreCase(DeletePolicy.IMMEDIATE.name())) {
+ deleteCurrentFile(fileToRoll);
+ } else {
+ // TODO: implement delay in the future
+ throw new IllegalArgumentException("Unsupported delete policy: " + deletePolicy);
+ }
}
- if (fileToRoll.length() != currentFile.get().getLength()) {
- String message = "File has changed size since being read: " + fileToRoll;
- throw new IllegalStateException(message);
+
+ /**
+ * Rename the given spooled file
+ *
+ * @param fileToRoll
+ * @throws IOException
+ */
+ private void rollCurrentFile(File fileToRoll) throws IOException {
+
+ File dest = new File(fileToRoll.getPath() + completedSuffix);
+ logger.info("Preparing to move file {} to {}", fileToRoll, dest);
+
+ // Before renaming, check whether destination file name exists
+ if (dest.exists() && PlatformDetect.isWindows()) {
+ /*
+ * If we are here, it means the completed file already exists. In almost
+ * every case this means the user is violating an assumption of Flume
+ * (that log files are placed in the spooling directory with unique
+ * names). However, there is a corner case on Windows systems where the
+ * file was already rolled but the rename was not atomic. If that seems
+ * likely, we let it pass with only a warning.
+ */
+ if (com.google.common.io.Files.equal(currentFile.get().getFile(), dest)) {
+ logger.warn("Completed file " + dest + " already exists, but files match, so continuing.");
+ boolean deleted = fileToRoll.delete();
+ if (!deleted) {
+ logger.error("Unable to delete file " + fileToRoll.getAbsolutePath()
+ + ". It will likely be ingested another time.");
+ sourceCounter.incrementGenericProcessingFail();
+ }
+ } else {
+ String message = "File name has been re-used with different"
+ + " files. Spooling assumptions violated for " + dest;
+ throw new IllegalStateException(message);
+ }
+
+ // Dest file exists and not on windows
+ } else if (dest.exists()) {
+ String message =
+ "File name has been re-used with different" + " files. Spooling assumptions violated for " + dest;
+ throw new IllegalStateException(message);
+
+ // Destination file does not already exist. We are good to go!
+ } else {
+ boolean renamed = fileToRoll.renameTo(dest);
+ if (renamed) {
+ logger.debug("Successfully rolled file {} to {}", fileToRoll, dest);
+
+ // now we no longer need the meta file
+ deleteMetaFile();
+ } else {
+ /* If we are here then the file cannot be renamed for a reason other
+ * than that the destination file exists (actually, that remains
+ * possible w/ small probability due to TOC-TOU conditions).*/
+ String message = "Unable to move " + fileToRoll + " to " + dest
+ + ". This will likely cause duplicate events. Please verify that "
+ + "flume has sufficient permissions to perform these operations.";
+ throw new FlumeException(message);
+ }
+ }
}
- if (deletePolicy.equalsIgnoreCase(DeletePolicy.NEVER.name())) {
- if (trackingPolicy == TrackingPolicy.RENAME) {
- rollCurrentFile(fileToRoll);
- } else {
- rollCurrentFileInTrackerDir(fileToRoll);
- }
- } else if (deletePolicy.equalsIgnoreCase(DeletePolicy.IMMEDIATE.name())) {
- deleteCurrentFile(fileToRoll);
- } else {
- // TODO: implement delay in the future
- throw new IllegalArgumentException("Unsupported delete policy: " +
- deletePolicy);
+ private void rollCurrentFileInTrackerDir(File fileToRoll) throws IOException {
+ Path path = fileToRoll.toPath();
+ Path relToRoll = getRelPathToSpoolDir(path);
+
+ File dest = new File(trackerDirectory.getPath(), relToRoll + completedSuffix);
+ logger.info("Preparing to create tracker file for {} at {}", fileToRoll, dest);
+ if (dest.exists()) {
+ String message =
+ "File name has been re-used with different" + " files. Spooling assumptions violated for " + dest;
+ throw new IllegalStateException(message);
+ }
+ // Create an empty file as an indicator
+ dest.getParentFile().mkdirs(); // create the parent dirs first
+ if (!dest.createNewFile()) {
+ throw new IOException("Could not create tracker file: " + dest);
+ }
}
- }
-
- /**
- * Rename the given spooled file
- *
- * @param fileToRoll
- * @throws IOException
- */
- private void rollCurrentFile(File fileToRoll) throws IOException {
-
- File dest = new File(fileToRoll.getPath() + completedSuffix);
- logger.info("Preparing to move file {} to {}", fileToRoll, dest);
-
- // Before renaming, check whether destination file name exists
- if (dest.exists() && PlatformDetect.isWindows()) {
- /*
- * If we are here, it means the completed file already exists. In almost
- * every case this means the user is violating an assumption of Flume
- * (that log files are placed in the spooling directory with unique
- * names). However, there is a corner case on Windows systems where the
- * file was already rolled but the rename was not atomic. If that seems
- * likely, we let it pass with only a warning.
- */
- if (com.google.common.io.Files.equal(currentFile.get().getFile(), dest)) {
- logger.warn("Completed file " + dest +
- " already exists, but files match, so continuing.");
- boolean deleted = fileToRoll.delete();
- if (!deleted) {
- logger.error("Unable to delete file " + fileToRoll.getAbsolutePath() +
- ". It will likely be ingested another time.");
- sourceCounter.incrementGenericProcessingFail();
- }
- } else {
- String message = "File name has been re-used with different" +
- " files. Spooling assumptions violated for " + dest;
- throw new IllegalStateException(message);
- }
-
- // Dest file exists and not on windows
- } else if (dest.exists()) {
- String message = "File name has been re-used with different" +
- " files. Spooling assumptions violated for " + dest;
- throw new IllegalStateException(message);
-
- // Destination file does not already exist. We are good to go!
- } else {
- boolean renamed = fileToRoll.renameTo(dest);
- if (renamed) {
- logger.debug("Successfully rolled file {} to {}", fileToRoll, dest);
+ /**
+ * Delete the given spooled file
+ *
+ * @param fileToDelete
+ * @throws IOException
+ */
+ private void deleteCurrentFile(File fileToDelete) throws IOException {
+ logger.info("Preparing to delete file {}", fileToDelete);
+ if (!fileToDelete.exists()) {
+ logger.warn("Unable to delete nonexistent file: {}", fileToDelete);
+ return;
+ }
+ if (!fileToDelete.delete()) {
+ throw new IOException("Unable to delete spool file: " + fileToDelete);
+ }
// now we no longer need the meta file
deleteMetaFile();
- } else {
- /* If we are here then the file cannot be renamed for a reason other
- * than that the destination file exists (actually, that remains
- * possible w/ small probability due to TOC-TOU conditions).*/
- String message = "Unable to move " + fileToRoll + " to " + dest +
- ". This will likely cause duplicate events. Please verify that " +
- "flume has sufficient permissions to perform these operations.";
- throw new FlumeException(message);
- }
- }
- }
-
- private void rollCurrentFileInTrackerDir(File fileToRoll) throws IOException {
- Path path = fileToRoll.toPath();
- Path relToRoll = getRelPathToSpoolDir(path);
-
- File dest = new File(trackerDirectory.getPath(), relToRoll + completedSuffix);
- logger.info("Preparing to create tracker file for {} at {}", fileToRoll, dest);
- if (dest.exists()) {
- String message = "File name has been re-used with different" +
- " files. Spooling assumptions violated for " + dest;
- throw new IllegalStateException(message);
- }
- //Create an empty file as an indicator
- dest.getParentFile().mkdirs(); //create the parent dirs first
- if (!dest.createNewFile()) {
- throw new IOException("Could not create tracker file: " + dest);
- }
- }
-
- /**
- * Delete the given spooled file
- *
- * @param fileToDelete
- * @throws IOException
- */
- private void deleteCurrentFile(File fileToDelete) throws IOException {
- logger.info("Preparing to delete file {}", fileToDelete);
- if (!fileToDelete.exists()) {
- logger.warn("Unable to delete nonexistent file: {}", fileToDelete);
- return;
- }
- if (!fileToDelete.delete()) {
- throw new IOException("Unable to delete spool file: " + fileToDelete);
- }
- // now we no longer need the meta file
- deleteMetaFile();
- }
-
- /**
- * Returns the next file to be consumed from the chosen directory.
- * If the directory is empty or the chosen file is not readable,
- * this will return an absent option.
- * If the {@link #consumeOrder} variable is {@link ConsumeOrder#OLDEST}
- * then returns the oldest file. If the {@link #consumeOrder} variable
- * is {@link ConsumeOrder#YOUNGEST} then returns the youngest file.
- * If two or more files are equally old/young, then the file name with
- * lower lexicographical value is returned.
- * If the {@link #consumeOrder} variable is {@link ConsumeOrder#RANDOM}
- * then cache the directory listing to amortize retreival cost, and return
- * any arbitary file from the directory.
- */
- private Optional<FileInfo> getNextFile() {
- List<File> candidateFiles = Collections.emptyList();
-
- if (consumeOrder != ConsumeOrder.RANDOM ||
- candidateFileIter == null ||
- !candidateFileIter.hasNext()) {
- candidateFiles = getCandidateFiles(spoolDirectory.toPath());
- listFilesCount++;
- candidateFileIter = candidateFiles.iterator();
}
- if (!candidateFileIter.hasNext()) { // No matching file in spooling directory.
- return Optional.absent();
- }
+ /**
+ * Returns the next file to be consumed from the chosen directory.
+ * If the directory is empty or the chosen file is not readable,
+ * this will return an absent option.
+ * If the {@link #consumeOrder} variable is {@link ConsumeOrder#OLDEST}
+ * then returns the oldest file. If the {@link #consumeOrder} variable
+ * is {@link ConsumeOrder#YOUNGEST} then returns the youngest file.
+ * If two or more files are equally old/young, then the file name with
+ * lower lexicographical value is returned.
+ * If the {@link #consumeOrder} variable is {@link ConsumeOrder#RANDOM}
+ * then cache the directory listing to amortize retreival cost, and return
+ * any arbitary file from the directory.
+ */
+ private Optional<FileInfo> getNextFile() {
+ List<File> candidateFiles = Collections.emptyList();
+
+ if (consumeOrder != ConsumeOrder.RANDOM || candidateFileIter == null || !candidateFileIter.hasNext()) {
+ candidateFiles = getCandidateFiles(spoolDirectory.toPath());
+ listFilesCount++;
+ candidateFileIter = candidateFiles.iterator();
+ }
- File selectedFile = candidateFileIter.next();
- if (consumeOrder == ConsumeOrder.RANDOM) { // Selected file is random.
- return openFile(selectedFile);
- } else if (consumeOrder == ConsumeOrder.YOUNGEST) {
- for (File candidateFile : candidateFiles) {
- long compare = selectedFile.lastModified() -
- candidateFile.lastModified();
- if (compare == 0) { // ts is same pick smallest lexicographically.
- selectedFile = smallerLexicographical(selectedFile, candidateFile);
- } else if (compare < 0) { // candidate is younger (cand-ts > selec-ts)
- selectedFile = candidateFile;
- }
- }
- } else { // default order is OLDEST
- for (File candidateFile : candidateFiles) {
- long compare = selectedFile.lastModified() -
- candidateFile.lastModified();
- if (compare == 0) { // ts is same pick smallest lexicographically.
- selectedFile = smallerLexicographical(selectedFile, candidateFile);
- } else if (compare > 0) { // candidate is older (cand-ts < selec-ts).
- selectedFile = candidateFile;
- }
- }
- }
+ if (!candidateFileIter.hasNext()) { // No matching file in spooling directory.
+ return Optional.absent();
+ }
- firstTimeRead = true;
+ File selectedFile = candidateFileIter.next();
+ if (consumeOrder == ConsumeOrder.RANDOM) { // Selected file is random.
+ return openFile(selectedFile);
+ } else if (consumeOrder == ConsumeOrder.YOUNGEST) {
+ for (File candidateFile : candidateFiles) {
+ long compare = selectedFile.lastModified() - candidateFile.lastModified();
+ if (compare == 0) { // ts is same pick smallest lexicographically.
+ selectedFile = smallerLexicographical(selectedFile, candidateFile);
+ } else if (compare < 0) { // candidate is younger (cand-ts > selec-ts)
+ selectedFile = candidateFile;
+ }
+ }
+ } else { // default order is OLDEST
+ for (File candidateFile : candidateFiles) {
+ long compare = selectedFile.lastModified() - candidateFile.lastModified();
+ if (compare == 0) { // ts is same pick smallest lexicographically.
+ selectedFile = smallerLexicographical(selectedFile, candidateFile);
+ } else if (compare > 0) { // candidate is older (cand-ts < selec-ts).
+ selectedFile = candidateFile;
+ }
+ }
+ }
- return openFile(selectedFile);
- }
+ firstTimeRead = true;
- private File smallerLexicographical(File f1, File f2) {
- if (f1.getName().compareTo(f2.getName()) < 0) {
- return f1;
- }
- return f2;
- }
-
- /**
- * Opens a file for consuming
- *
- * @param file
- * @return {@link FileInfo} for the file to consume or absent option if the
- * file does not exists or readable.
- */
- private Optional<FileInfo> openFile(File file) {
- try {
- // roll the meta file, if needed
- String nextPath = file.getPath();
- PositionTracker tracker =
- DurablePositionTracker.getInstance(metaFile, nextPath);
- if (!tracker.getTarget().equals(nextPath)) {
- tracker.close();
- deleteMetaFile();
- tracker = DurablePositionTracker.getInstance(metaFile, nextPath);
- }
-
- // sanity check
- Preconditions.checkState(tracker.getTarget().equals(nextPath),
- "Tracker target %s does not equal expected filename %s",
- tracker.getTarget(), nextPath);
-
- ResettableInputStream in =
- new ResettableFileInputStream(file, tracker,
- ResettableFileInputStream.DEFAULT_BUF_SIZE, inputCharset,
- decodeErrorPolicy);
- EventDeserializer deserializer =
- EventDeserializerFactory.getInstance(deserializerType, deserializerContext, in);
-
- return Optional.of(new FileInfo(file, deserializer));
- } catch (FileNotFoundException e) {
- // File could have been deleted in the interim
- logger.warn("Could not find file: " + file, e);
- return Optional.absent();
- } catch (IOException e) {
- logger.error("Exception opening file: " + file, e);
- sourceCounter.incrementGenericProcessingFail();
- return Optional.absent();
+ return openFile(selectedFile);
}
- }
- private void deleteMetaFile() throws IOException {
- if (metaFile.exists() && !metaFile.delete()) {
- throw new IOException("Unable to delete old meta file " + metaFile);
- }
- }
-
- /**
- * An immutable class with information about a file being processed.
- */
- private static class FileInfo {
- private final File file;
- private final long length;
- private final long lastModified;
- private final EventDeserializer deserializer;
-
- public FileInfo(File file, EventDeserializer deserializer) {
- this.file = file;
- this.length = file.length();
- this.lastModified = file.lastModified();
- this.deserializer = deserializer;
+ private File smallerLexicographical(File f1, File f2) {
+ if (f1.getName().compareTo(f2.getName()) < 0) {
+ return f1;
+ }
+ return f2;
+ }
+
+ /**
+ * Opens a file for consuming
+ *
+ * @param file
+ * @return {@link FileInfo} for the file to consume or absent option if the
+ * file does not exists or readable.
+ */
+ private Optional<FileInfo> openFile(File file) {
+ try {
+ // roll the meta file, if needed
+ String nextPath = file.getPath();
+ PositionTracker tracker = DurablePositionTracker.getInstance(metaFile, nextPath);
+ if (!tracker.getTarget().equals(nextPath)) {
+ tracker.close();
+ deleteMetaFile();
+ tracker = DurablePositionTracker.getInstance(metaFile, nextPath);
+ }
+
+ // sanity check
+ Preconditions.checkState(
+ tracker.getTarget().equals(nextPath),
+ "Tracker target %s does not equal expected filename %s",
+ tracker.getTarget(),
+ nextPath);
+
+ ResettableInputStream in = new ResettableFileInputStream(
+ file, tracker, ResettableFileInputStream.DEFAULT_BUF_SIZE, inputCharset, decodeErrorPolicy);
+ EventDeserializer deserializer =
+ EventDeserializerFactory.getInstance(deserializerType, deserializerContext, in);
+
+ return Optional.of(new FileInfo(file, deserializer));
+ } catch (FileNotFoundException e) {
+ // File could have been deleted in the interim
+ logger.warn("Could not find file: " + file, e);
+ return Optional.absent();
+ } catch (IOException e) {
+ logger.error("Exception opening file: " + file, e);
+ sourceCounter.incrementGenericProcessingFail();
+ return Optional.absent();
+ }
}
- public long getLength() {
- return length;
+ private void deleteMetaFile() throws IOException {
+ if (metaFile.exists() && !metaFile.delete()) {
+ throw new IOException("Unable to delete old meta file " + metaFile);
+ }
}
- public long getLastModified() {
- return lastModified;
- }
+ /**
+ * An immutable class with information about a file being processed.
+ */
+ private static class FileInfo {
+ private final File file;
+ private final long length;
+ private final long lastModified;
+ private final EventDeserializer deserializer;
+
+ public FileInfo(File file, EventDeserializer deserializer) {
+ this.file = file;
+ this.length = file.length();
+ this.lastModified = file.lastModified();
+ this.deserializer = deserializer;
+ }
- public EventDeserializer getDeserializer() {
- return deserializer;
- }
+ public long getLength() {
+ return length;
+ }
- public File getFile() {
- return file;
- }
- }
-
- @InterfaceAudience.Private
- @InterfaceStability.Unstable
- enum DeletePolicy {
- NEVER,
- IMMEDIATE,
- DELAY
- }
-
- @InterfaceAudience.Private
- @InterfaceStability.Unstable
- public enum TrackingPolicy {
- RENAME,
- TRACKER_DIR
- }
-
-
- /**
- * Special builder class for ReliableSpoolingFileEventReader
- */
- public static class Builder {
- private File spoolDirectory;
- private String completedSuffix =
- SpoolDirectorySourceConfigurationConstants.SPOOLED_FILE_SUFFIX;
- private String includePattern =
- SpoolDirectorySourceConfigurationConstants.DEFAULT_INCLUDE_PAT;
- private String ignorePattern =
- SpoolDirectorySourceConfigurationConstants.DEFAULT_IGNORE_PAT;
- private String trackerDirPath =
- SpoolDirectorySourceConfigurationConstants.DEFAULT_TRACKER_DIR;
- private Boolean annotateFileName =
- SpoolDirectorySourceConfigurationConstants.DEFAULT_FILE_HEADER;
- private String fileNameHeader =
- SpoolDirectorySourceConfigurationConstants.DEFAULT_FILENAME_HEADER_KEY;
- private Boolean annotateBaseName =
- SpoolDirectorySourceConfigurationConstants.DEFAULT_BASENAME_HEADER;
- private String baseNameHeader =
- SpoolDirectorySourceConfigurationConstants.DEFAULT_BASENAME_HEADER_KEY;
- private String deserializerType =
- SpoolDirectorySourceConfigurationConstants.DEFAULT_DESERIALIZER;
- private Context deserializerContext = new Context();
- private String deletePolicy =
- SpoolDirectorySourceConfigurationConstants.DEFAULT_DELETE_POLICY;
- private String trackingPolicy =
- SpoolDirectorySourceConfigurationConstants.DEFAULT_TRACKING_POLICY;
- private String inputCharset =
- SpoolDirectorySourceConfigurationConstants.DEFAULT_INPUT_CHARSET;
- private DecodeErrorPolicy decodeErrorPolicy = DecodeErrorPolicy.valueOf(
- SpoolDirectorySourceConfigurationConstants.DEFAULT_DECODE_ERROR_POLICY
- .toUpperCase(Locale.ENGLISH));
- private ConsumeOrder consumeOrder =
- SpoolDirectorySourceConfigurationConstants.DEFAULT_CONSUME_ORDER;
- private boolean recursiveDirectorySearch =
- SpoolDirectorySourceConfigurationConstants.DEFAULT_RECURSIVE_DIRECTORY_SEARCH;
- private SourceCounter sourceCounter;
-
- public Builder spoolDirectory(File directory) {
- this.spoolDirectory = directory;
- return this;
- }
+ public long getLastModified() {
+ return lastModified;
+ }
- public Builder completedSuffix(String completedSuffix) {
- this.completedSuffix = completedSuffix;
- return this;
- }
+ public EventDeserializer getDeserializer() {
+ return deserializer;
+ }
- public Builder includePattern(String includePattern) {
- this.includePattern = includePattern;
- return this;
+ public File getFile() {
+ return file;
+ }
}
- public Builder ignorePattern(String ignorePattern) {
- this.ignorePattern = ignorePattern;
- return this;
- }
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ enum DeletePolicy {
+ NEVER,
+ IMMEDIATE,
+ DELAY
+ }
+
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ public enum TrackingPolicy {
+ RENAME,
+ TRACKER_DIR
+ }
+
+ /**
+ * Special builder class for ReliableSpoolingFileEventReader
+ */
+ public static class Builder {
+ private File spoolDirectory;
+ private String completedSuffix = SpoolDirectorySourceConfigurationConstants.SPOOLED_FILE_SUFFIX;
+ private String includePattern = SpoolDirectorySourceConfigurationConstants.DEFAULT_INCLUDE_PAT;
+ private String ignorePattern = SpoolDirectorySourceConfigurationConstants.DEFAULT_IGNORE_PAT;
+ private String trackerDirPath = SpoolDirectorySourceConfigurationConstants.DEFAULT_TRACKER_DIR;
+ private Boolean annotateFileName = SpoolDirectorySourceConfigurationConstants.DEFAULT_FILE_HEADER;
+ private String fileNameHeader = SpoolDirectorySourceConfigurationConstants.DEFAULT_FILENAME_HEADER_KEY;
+ private Boolean annotateBaseName = SpoolDirectorySourceConfigurationConstants.DEFAULT_BASENAME_HEADER;
+ private String baseNameHeader = SpoolDirectorySourceConfigurationConstants.DEFAULT_BASENAME_HEADER_KEY;
+ private String deserializerType = SpoolDirectorySourceConfigurationConstants.DEFAULT_DESERIALIZER;
+ private Context deserializerContext = new Context();
+ private String deletePolicy = SpoolDirectorySourceConfigurationConstants.DEFAULT_DELETE_POLICY;
+ private String trackingPolicy = SpoolDirectorySourceConfigurationConstants.DEFAULT_TRACKING_POLICY;
+ private String inputCharset = SpoolDirectorySourceConfigurationConstants.DEFAULT_INPUT_CHARSET;
+ private DecodeErrorPolicy decodeErrorPolicy = DecodeErrorPolicy.valueOf(
+ SpoolDirectorySourceConfigurationConstants.DEFAULT_DECODE_ERROR_POLICY.toUpperCase(Locale.ENGLISH));
+ private ConsumeOrder consumeOrder = SpoolDirectorySourceConfigurationConstants.DEFAULT_CONSUME_ORDER;
+ private boolean recursiveDirectorySearch =
+ SpoolDirectorySourceConfigurationConstants.DEFAULT_RECURSIVE_DIRECTORY_SEARCH;
+ private SourceCounter sourceCounter;
+
+ public Builder spoolDirectory(File directory) {
+ this.spoolDirectory = directory;
+ return this;
+ }
- public Builder trackerDirPath(String trackerDirPath) {
- this.trackerDirPath = trackerDirPath;
- return this;
- }
+ public Builder completedSuffix(String completedSuffix) {
+ this.completedSuffix = completedSuffix;
+ return this;
+ }
- public Builder annotateFileName(Boolean annotateFileName) {
- this.annotateFileName = annotateFileName;
- return this;
- }
+ public Builder includePattern(String includePattern) {
+ this.includePattern = includePattern;
+ return this;
+ }
- public Builder fileNameHeader(String fileNameHeader) {
- this.fileNameHeader = fileNameHeader;
- return this;
- }
+ public Builder ignorePattern(String ignorePattern) {
+ this.ignorePattern = ignorePattern;
+ return this;
+ }
- public Builder annotateBaseName(Boolean annotateBaseName) {
- this.annotateBaseName = annotateBaseName;
- return this;
- }
+ public Builder trackerDirPath(String trackerDirPath) {
+ this.trackerDirPath = trackerDirPath;
+ return this;
+ }
- public Builder baseNameHeader(String baseNameHeader) {
- this.baseNameHeader = baseNameHeader;
- return this;
- }
+ public Builder annotateFileName(Boolean annotateFileName) {
+ this.annotateFileName = annotateFileName;
+ return this;
+ }
- public Builder deserializerType(String deserializerType) {
- this.deserializerType = deserializerType;
- return this;
- }
+ public Builder fileNameHeader(String fileNameHeader) {
+ this.fileNameHeader = fileNameHeader;
+ return this;
+ }
- public Builder deserializerContext(Context deserializerContext) {
- this.deserializerContext = deserializerContext;
- return this;
- }
+ public Builder annotateBaseName(Boolean annotateBaseName) {
+ this.annotateBaseName = annotateBaseName;
+ return this;
+ }
- public Builder deletePolicy(String deletePolicy) {
- this.deletePolicy = deletePolicy;
- return this;
- }
+ public Builder baseNameHeader(String baseNameHeader) {
+ this.baseNameHeader = baseNameHeader;
+ return this;
+ }
- public Builder trackingPolicy(String trackingPolicy) {
- this.trackingPolicy = trackingPolicy;
- return this;
- }
+ public Builder deserializerType(String deserializerType) {
+ this.deserializerType = deserializerType;
+ return this;
+ }
- public Builder inputCharset(String inputCharset) {
- this.inputCharset = inputCharset;
- return this;
- }
+ public Builder deserializerContext(Context deserializerContext) {
+ this.deserializerContext = deserializerContext;
+ return this;
+ }
- public Builder recursiveDirectorySearch(boolean recursiveDirectorySearch) {
- this.recursiveDirectorySearch = recursiveDirectorySearch;
- return this;
- }
+ public Builder deletePolicy(String deletePolicy) {
+ this.deletePolicy = deletePolicy;
+ return this;
+ }
- public Builder decodeErrorPolicy(DecodeErrorPolicy decodeErrorPolicy) {
- this.decodeErrorPolicy = decodeErrorPolicy;
- return this;
- }
+ public Builder trackingPolicy(String trackingPolicy) {
+ this.trackingPolicy = trackingPolicy;
+ return this;
+ }
- public Builder consumeOrder(ConsumeOrder consumeOrder) {
- this.consumeOrder = consumeOrder;
- return this;
- }
+ public Builder inputCharset(String inputCharset) {
+ this.inputCharset = inputCharset;
+ return this;
+ }
- public Builder sourceCounter(SourceCounter sourceCounter) {
- this.sourceCounter = sourceCounter;
- return this;
- }
+ public Builder recursiveDirectorySearch(boolean recursiveDirectorySearch) {
+ this.recursiveDirectorySearch = recursiveDirectorySearch;
+ return this;
+ }
- public ReliableSpoolingFileEventReader build() throws IOException {
- return new ReliableSpoolingFileEventReader(spoolDirectory, completedSuffix,
- includePattern, ignorePattern, trackerDirPath, annotateFileName, fileNameHeader,
- annotateBaseName, baseNameHeader, deserializerType,
- deserializerContext, deletePolicy, trackingPolicy, inputCharset, decodeErrorPolicy,
- consumeOrder, recursiveDirectorySearch, sourceCounter);
- }
- }
+ public Builder decodeErrorPolicy(DecodeErrorPolicy decodeErrorPolicy) {
+ this.decodeErrorPolicy = decodeErrorPolicy;
+ return this;
+ }
+
+ public Builder consumeOrder(ConsumeOrder consumeOrder) {
+ this.consumeOrder = consumeOrder;
+ return this;
+ }
+
+ public Builder sourceCounter(SourceCounter sourceCounter) {
+ this.sourceCounter = sourceCounter;
+ return this;
+ }
+ public ReliableSpoolingFileEventReader build() throws IOException {
+ return new ReliableSpoolingFileEventReader(
+ spoolDirectory,
+ completedSuffix,
+ includePattern,
+ ignorePattern,
+ trackerDirPath,
+ annotateFileName,
+ fileNameHeader,
+ annotateBaseName,
+ baseNameHeader,
+ deserializerType,
+ deserializerContext,
+ deletePolicy,
+ trackingPolicy,
+ inputCharset,
+ decodeErrorPolicy,
+ consumeOrder,
+ recursiveDirectorySearch,
+ sourceCounter);
+ }
+ }
}
diff --git a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/sink/AvroSink.java b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/sink/AvroSink.java
index 4b9d6c2..6f7d480 100644
--- a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/sink/AvroSink.java
+++ b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/sink/AvroSink.java
@@ -1,24 +1,22 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
-
package org.apache.flume.rpc.avro.sink;
+import java.util.Properties;
import org.apache.flume.Channel;
import org.apache.flume.Sink;
import org.apache.flume.api.RpcClient;
@@ -28,8 +26,6 @@ import org.apache.flume.sink.AbstractRpcSink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Properties;
-
/**
* <p>
* A {@link Sink} implementation that can send events to an RPC server (such as
@@ -107,11 +103,11 @@ import java.util.Properties;
*/
public class AvroSink extends AbstractRpcSink {
- private static final Logger logger = LoggerFactory.getLogger(AvroSink.class);
+ private static final Logger logger = LoggerFactory.getLogger(AvroSink.class);
- @Override
- protected RpcClient initializeRpcClient(Properties props) {
- logger.info("Attempting to create Avro Rpc client.");
- return RpcClientFactory.getInstance(props);
- }
+ @Override
+ protected RpcClient initializeRpcClient(Properties props) {
+ logger.info("Attempting to create Avro Rpc client.");
+ return RpcClientFactory.getInstance(props);
+ }
}
diff --git a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/source/AvroSource.java b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/source/AvroSource.java
index 23346cb..a46411c 100644
--- a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/source/AvroSource.java
+++ b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/source/AvroSource.java
@@ -1,22 +1,19 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
-
package org.apache.flume.rpc.avro.source;
import io.netty.channel.ChannelPipeline;
@@ -27,6 +24,16 @@ import io.netty.handler.ipfilter.IpFilterRule;
import io.netty.handler.ipfilter.IpFilterRuleType;
import io.netty.handler.ipfilter.RuleBasedIpFilter;
import io.netty.handler.ssl.SslHandler;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import javax.net.ssl.SSLEngine;
import org.apache.avro.ipc.Responder;
import org.apache.avro.ipc.Server;
import org.apache.avro.ipc.netty.NettyServer;
@@ -51,17 +58,6 @@ import org.apache.flume.source.avro.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.net.ssl.SSLEngine;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
/**
* <p>
* A {@link Source} implementation that receives Avro events from clients that
@@ -114,257 +110,246 @@ import java.util.concurrent.TimeUnit;
* <b>Metrics</b>
* </p>
*/
-public class AvroSource extends SslContextAwareAbstractSource implements EventDrivenSource,
- Configurable, AvroSourceProtocol {
-
- private static final String THREADS = "threads";
-
- private static final Logger logger = LoggerFactory.getLogger(AvroSource.class);
-
- private static final String PORT_KEY = "port";
- private static final String BIND_KEY = "bind";
- private static final String COMPRESSION_TYPE = "compression-type";
- private static final String IP_FILTER_KEY = "ipFilter";
- private static final String IP_FILTER_RULES_KEY = "ipFilterRules";
- private int port;
- private String bindAddress;
- private String compressionType;
- private boolean enableIpFilter;
- private String patternRuleConfigDefinition;
-
- private Server server;
- private SourceCounter sourceCounter;
-
- private int maxThreads;
- private ScheduledExecutorService connectionCountUpdater;
-
- private List<IpFilterRule> rules;
-
- @Override
- public void configure(Context context) {
- configureSsl(context);
- Configurables.ensureRequiredNonNull(context, PORT_KEY, BIND_KEY);
-
- port = context.getInteger(PORT_KEY);
- bindAddress = context.getString(BIND_KEY);
- compressionType = context.getString(COMPRESSION_TYPE, "none");
-
- try {
- maxThreads = context.getInteger(THREADS, 0);
- } catch (NumberFormatException e) {
- logger.warn("AVRO source\'s \"threads\" property must specify an integer value.",
- context.getString(THREADS));
+public class AvroSource extends SslContextAwareAbstractSource
+ implements EventDrivenSource, Configurable, AvroSourceProtocol {
+
+ private static final String THREADS = "threads";
+
+ private static final Logger logger = LoggerFactory.getLogger(AvroSource.class);
+
+ private static final String PORT_KEY = "port";
+ private static final String BIND_KEY = "bind";
+ private static final String COMPRESSION_TYPE = "compression-type";
+ private static final String IP_FILTER_KEY = "ipFilter";
+ private static final String IP_FILTER_RULES_KEY = "ipFilterRules";
+ private int port;
+ private String bindAddress;
+ private String compressionType;
+ private boolean enableIpFilter;
+ private String patternRuleConfigDefinition;
+
+ private Server server;
+ private SourceCounter sourceCounter;
+
+ private int maxThreads;
+ private ScheduledExecutorService connectionCountUpdater;
+
+ private List<IpFilterRule> rules;
+
+ @Override
+ public void configure(Context context) {
+ configureSsl(context);
+ Configurables.ensureRequiredNonNull(context, PORT_KEY, BIND_KEY);
+
+ port = context.getInteger(PORT_KEY);
+ bindAddress = context.getString(BIND_KEY);
+ compressionType = context.getString(COMPRESSION_TYPE, "none");
+
+ try {
+ maxThreads = context.getInteger(THREADS, 0);
+ } catch (NumberFormatException e) {
+ logger.warn(
+ "AVRO source\'s \"threads\" property must specify an integer value.", context.getString(THREADS));
+ }
+
+ enableIpFilter = context.getBoolean(IP_FILTER_KEY, false);
+ if (enableIpFilter) {
+ patternRuleConfigDefinition = context.getString(IP_FILTER_RULES_KEY);
+ if (patternRuleConfigDefinition == null
+ || patternRuleConfigDefinition.trim().isEmpty()) {
+ throw new FlumeException("ipFilter is configured with true but ipFilterRules is not defined:" + " ");
+ }
+ String[] patternRuleDefinitions = patternRuleConfigDefinition.split(",");
+ rules = new ArrayList<IpFilterRule>(patternRuleDefinitions.length);
+ for (String patternRuleDefinition : patternRuleDefinitions) {
+ rules.add(generateRule(patternRuleDefinition));
+ }
+ }
+
+ if (sourceCounter == null) {
+ sourceCounter = new SourceCounter(getName());
+ }
}
- enableIpFilter = context.getBoolean(IP_FILTER_KEY, false);
- if (enableIpFilter) {
- patternRuleConfigDefinition = context.getString(IP_FILTER_RULES_KEY);
- if (patternRuleConfigDefinition == null ||
- patternRuleConfigDefinition.trim().isEmpty()) {
- throw new FlumeException(
- "ipFilter is configured with true but ipFilterRules is not defined:" +
- " ");
- }
- String[] patternRuleDefinitions = patternRuleConfigDefinition.split(
- ",");
- rules = new ArrayList<IpFilterRule>(patternRuleDefinitions.length);
- for (String patternRuleDefinition : patternRuleDefinitions) {
- rules.add(generateRule(patternRuleDefinition));
- }
- }
+ @Override
+ public void start() {
+ logger.info("Starting {}...", this);
- if (sourceCounter == null) {
- sourceCounter = new SourceCounter(getName());
- }
- }
-
- @Override
- public void start() {
- logger.info("Starting {}...", this);
+ try {
+ Responder responder = new SpecificResponder(AvroSourceProtocol.class, this);
+ boolean enableCompression = compressionType.equalsIgnoreCase("deflate");
- try {
- Responder responder = new SpecificResponder(AvroSourceProtocol.class, this);
- boolean enableCompression = compressionType.equalsIgnoreCase("deflate");
-
- server = new NettyServer(responder, new InetSocketAddress(bindAddress, port),
- (ch) -> {
+ server = new NettyServer(responder, new InetSocketAddress(bindAddress, port), (ch) -> {
ChannelPipeline pipeline = ch.pipeline();
if (enableCompression) {
- ZlibEncoder encoder = new JZlibEncoder(6);
- pipeline.addFirst("deflater", encoder);
- pipeline.addFirst("inflater", new JZlibDecoder());
+ ZlibEncoder encoder = new JZlibEncoder(6);
+ pipeline.addFirst("deflater", encoder);
+ pipeline.addFirst("inflater", new JZlibDecoder());
}
Optional<SSLEngine> engine = getSslEngine(false);
engine.ifPresent(sslEngine -> pipeline.addLast("ssl", new SslHandler(sslEngine)));
if (enableIpFilter) {
- logger.info("Setting up ipFilter with the following rule definition: " +
- patternRuleConfigDefinition);
- RuleBasedIpFilter filter = new RuleBasedIpFilter(rules.toArray(new IpFilterRule[0]));
- logger.info("Adding ipFilter with " + rules.size() + " rules");
- pipeline.addFirst("ipFilter", filter);
+ logger.info(
+ "Setting up ipFilter with the following rule definition: " + patternRuleConfigDefinition);
+ RuleBasedIpFilter filter = new RuleBasedIpFilter(rules.toArray(new IpFilterRule[0]));
+ logger.info("Adding ipFilter with " + rules.size() + " rules");
+ pipeline.addFirst("ipFilter", filter);
}
- });
- } catch (Exception nce) {
- logger.error("Avro source {} startup failed. Cannot initialize Netty server", getName(), nce);
- stop();
- throw new FlumeException("Failed to set up server socket", nce);
+ });
+ } catch (Exception nce) {
+ logger.error("Avro source {} startup failed. Cannot initialize Netty server", getName(), nce);
+ stop();
+ throw new FlumeException("Failed to set up server socket", nce);
+ }
+
+ connectionCountUpdater = Executors.newSingleThreadScheduledExecutor();
+ server.start();
+ sourceCounter.start();
+ super.start();
+ final NettyServer srv = (NettyServer) server;
+ connectionCountUpdater.scheduleWithFixedDelay(
+ () -> sourceCounter.setOpenConnectionCount(Long.valueOf(srv.getNumActiveConnections())),
+ 0,
+ 60,
+ TimeUnit.SECONDS);
+
+ logger.info("Avro source {} started.", getName());
}
- connectionCountUpdater = Executors.newSingleThreadScheduledExecutor();
- server.start();
- sourceCounter.start();
- super.start();
- final NettyServer srv = (NettyServer)server;
- connectionCountUpdater.scheduleWithFixedDelay(
- () -> sourceCounter.setOpenConnectionCount(Long.valueOf(srv.getNumActiveConnections())),
- 0, 60, TimeUnit.SECONDS);
-
- logger.info("Avro source {} started.", getName());
- }
-
- @Override
- public void stop() {
- logger.info("Avro source {} stopping: {}", getName(), this);
-
- if (server != null) {
- server.close();
- try {
- server.join();
- server = null;
- } catch (InterruptedException e) {
- logger.info("Avro source " + getName() + ": Interrupted while waiting " +
- "for Avro server to stop. Exiting. Exception follows.", e);
- Thread.currentThread().interrupt();
- }
+ @Override
+ public void stop() {
+ logger.info("Avro source {} stopping: {}", getName(), this);
+
+ if (server != null) {
+ server.close();
+ try {
+ server.join();
+ server = null;
+ } catch (InterruptedException e) {
+ logger.info(
+ "Avro source " + getName() + ": Interrupted while waiting "
+ + "for Avro server to stop. Exiting. Exception follows.",
+ e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ sourceCounter.stop();
+ if (connectionCountUpdater != null) {
+ connectionCountUpdater.shutdownNow();
+ connectionCountUpdater = null;
+ }
+
+ super.stop();
+ logger.info("Avro source {} stopped. Metrics: {}", getName(), sourceCounter);
}
- sourceCounter.stop();
- if (connectionCountUpdater != null) {
- connectionCountUpdater.shutdownNow();
- connectionCountUpdater = null;
+ @Override
+ public String toString() {
+ return "Avro source " + getName() + ": { bindAddress: " + bindAddress + ", port: " + port + " }";
}
- super.stop();
- logger.info("Avro source {} stopped. Metrics: {}", getName(), sourceCounter);
- }
-
- @Override
- public String toString() {
- return "Avro source " + getName() + ": { bindAddress: " + bindAddress +
- ", port: " + port + " }";
- }
-
- /**
- * Helper function to convert a map of CharSequence to a map of String.
- */
- private static Map<String, String> toStringMap(Map<CharSequence, CharSequence> charSeqMap) {
- Map<String, String> stringMap = new HashMap<>();
- for (Map.Entry<CharSequence, CharSequence> entry : charSeqMap.entrySet()) {
- stringMap.put(entry.getKey().toString(), entry.getValue().toString());
- }
- return stringMap;
- }
-
- @Override
- public Status append(AvroFlumeEvent avroEvent) {
- if (logger.isDebugEnabled()) {
- if (LogPrivacyUtil.allowLogRawData()) {
- logger.debug("Avro source {}: Received avro event: {}", getName(), avroEvent);
- } else {
- logger.debug("Avro source {}: Received avro event", getName());
- }
+ /**
+ * Helper function to convert a map of CharSequence to a map of String.
+ */
+ private static Map<String, String> toStringMap(Map<CharSequence, CharSequence> charSeqMap) {
+ Map<String, String> stringMap = new HashMap<>();
+ for (Map.Entry<CharSequence, CharSequence> entry : charSeqMap.entrySet()) {
+ stringMap.put(entry.getKey().toString(), entry.getValue().toString());
+ }
+ return stringMap;
}
- sourceCounter.incrementAppendReceivedCount();
- sourceCounter.incrementEventReceivedCount();
-
- Event event = EventBuilder.withBody(avroEvent.getBody().array(),
- toStringMap(avroEvent.getHeaders()));
-
- try {
- getChannelProcessor().processEvent(event);
- } catch (ChannelException ex) {
- logger.warn("Avro source " + getName() + ": Unable to process event. " +
- "Exception follows.", ex);
- sourceCounter.incrementChannelWriteFail();
- return Status.FAILED;
+ @Override
+ public Status append(AvroFlumeEvent avroEvent) {
+ if (logger.isDebugEnabled()) {
+ if (LogPrivacyUtil.allowLogRawData()) {
+ logger.debug("Avro source {}: Received avro event: {}", getName(), avroEvent);
+ } else {
+ logger.debug("Avro source {}: Received avro event", getName());
+ }
+ }
+
+ sourceCounter.incrementAppendReceivedCount();
+ sourceCounter.incrementEventReceivedCount();
+
+ Event event = EventBuilder.withBody(avroEvent.getBody().array(), toStringMap(avroEvent.getHeaders()));
+
+ try {
+ getChannelProcessor().processEvent(event);
+ } catch (ChannelException ex) {
+ logger.warn("Avro source " + getName() + ": Unable to process event. " + "Exception follows.", ex);
+ sourceCounter.incrementChannelWriteFail();
+ return Status.FAILED;
+ }
+
+ sourceCounter.incrementAppendAcceptedCount();
+ sourceCounter.incrementEventAcceptedCount();
+
+ return Status.OK;
}
- sourceCounter.incrementAppendAcceptedCount();
- sourceCounter.incrementEventAcceptedCount();
+ @Override
+ public Status appendBatch(List<AvroFlumeEvent> events) {
+ logger.debug("Avro source {}: Received avro event batch of {} events.", getName(), events.size());
+ sourceCounter.incrementAppendBatchReceivedCount();
+ sourceCounter.addToEventReceivedCount(events.size());
- return Status.OK;
- }
+ List<Event> batch = new ArrayList<Event>();
- @Override
- public Status appendBatch(List<AvroFlumeEvent> events) {
- logger.debug("Avro source {}: Received avro event batch of {} events.",
- getName(), events.size());
- sourceCounter.incrementAppendBatchReceivedCount();
- sourceCounter.addToEventReceivedCount(events.size());
+ for (AvroFlumeEvent avroEvent : events) {
+ Event event = EventBuilder.withBody(avroEvent.getBody().array(), toStringMap(avroEvent.getHeaders()));
- List<Event> batch = new ArrayList<Event>();
+ batch.add(event);
+ }
- for (AvroFlumeEvent avroEvent : events) {
- Event event = EventBuilder.withBody(avroEvent.getBody().array(),
- toStringMap(avroEvent.getHeaders()));
+ try {
+ getChannelProcessor().processEventBatch(batch);
+ } catch (Throwable t) {
+ logger.error("Avro source " + getName() + ": Unable to process event " + "batch. Exception follows.", t);
+ sourceCounter.incrementChannelWriteFail();
+ if (t instanceof Error) {
+ throw (Error) t;
+ }
+ return Status.FAILED;
+ }
- batch.add(event);
- }
+ sourceCounter.incrementAppendBatchAcceptedCount();
+ sourceCounter.addToEventAcceptedCount(events.size());
- try {
- getChannelProcessor().processEventBatch(batch);
- } catch (Throwable t) {
- logger.error("Avro source " + getName() + ": Unable to process event " +
- "batch. Exception follows.", t);
- sourceCounter.incrementChannelWriteFail();
- if (t instanceof Error) {
- throw (Error) t;
- }
- return Status.FAILED;
+ return Status.OK;
}
- sourceCounter.incrementAppendBatchAcceptedCount();
- sourceCounter.addToEventAcceptedCount(events.size());
-
- return Status.OK;
- }
-
- private PatternRule generateRule(String patternRuleDefinition) throws FlumeException {
- patternRuleDefinition = patternRuleDefinition.trim();
- //first validate the format
- int firstColonIndex = patternRuleDefinition.indexOf(":");
- if (firstColonIndex == -1) {
- throw new FlumeException(
- "Invalid ipFilter patternRule '" + patternRuleDefinition +
- "' should look like <'allow' or 'deny'>:<'ip' or " +
- "'name'>:<pattern>");
- } else {
- String ruleAccessFlag = patternRuleDefinition.substring(0, firstColonIndex);
- int secondColonIndex = patternRuleDefinition.indexOf(":", firstColonIndex + 1);
- if ((!ruleAccessFlag.equals("allow") && !ruleAccessFlag.equals("deny")) ||
- secondColonIndex == -1) {
- throw new FlumeException(
- "Invalid ipFilter patternRule '" + patternRuleDefinition +
- "' should look like <'allow' or 'deny'>:<'ip' or " +
- "'name'>:<pattern>");
- }
-
- String patternTypeFlag = patternRuleDefinition.substring(
- firstColonIndex + 1, secondColonIndex);
- if ((!patternTypeFlag.equals("ip") && !patternTypeFlag.equals("name"))) {
- throw new FlumeException(
- "Invalid ipFilter patternRule '" + patternRuleDefinition +
- "' should look like <'allow' or 'deny'>:<'ip' or " +
- "'name'>:<pattern>");
- }
-
- boolean isAllow = ruleAccessFlag.equals("allow");
- String patternRuleString = (patternTypeFlag.equals("ip") ? "i" : "n")
- + ":" + patternRuleDefinition.substring(secondColonIndex + 1);
- IpFilterRuleType ruleType = isAllow ? IpFilterRuleType.ACCEPT : IpFilterRuleType.REJECT;
- logger.info("Adding ipFilter PatternRule: {} {}", ruleType, patternRuleString);
- return new PatternRule(ruleType, patternRuleString);
+ private PatternRule generateRule(String patternRuleDefinition) throws FlumeException {
+ patternRuleDefinition = patternRuleDefinition.trim();
+ // first validate the format
+ int firstColonIndex = patternRuleDefinition.indexOf(":");
+ if (firstColonIndex == -1) {
+ throw new FlumeException("Invalid ipFilter patternRule '" + patternRuleDefinition
+ + "' should look like <'allow' or 'deny'>:<'ip' or "
+ + "'name'>:<pattern>");
+ } else {
+ String ruleAccessFlag = patternRuleDefinition.substring(0, firstColonIndex);
+ int secondColonIndex = patternRuleDefinition.indexOf(":", firstColonIndex + 1);
+ if ((!ruleAccessFlag.equals("allow") && !ruleAccessFlag.equals("deny")) || secondColonIndex == -1) {
+ throw new FlumeException("Invalid ipFilter patternRule '" + patternRuleDefinition
+ + "' should look like <'allow' or 'deny'>:<'ip' or "
+ + "'name'>:<pattern>");
+ }
+
+ String patternTypeFlag = patternRuleDefinition.substring(firstColonIndex + 1, secondColonIndex);
+ if ((!patternTypeFlag.equals("ip") && !patternTypeFlag.equals("name"))) {
+ throw new FlumeException("Invalid ipFilter patternRule '" + patternRuleDefinition
+ + "' should look like <'allow' or 'deny'>:<'ip' or "
+ + "'name'>:<pattern>");
+ }
+
+ boolean isAllow = ruleAccessFlag.equals("allow");
+ String patternRuleString = (patternTypeFlag.equals("ip") ? "i" : "n") + ":"
+ + patternRuleDefinition.substring(secondColonIndex + 1);
+ IpFilterRuleType ruleType = isAllow ? IpFilterRuleType.ACCEPT : IpFilterRuleType.REJECT;
+ logger.info("Adding ipFilter PatternRule: {} {}", ruleType, patternRuleString);
+ return new PatternRule(ruleType, patternRuleString);
+ }
}
- }
}
diff --git a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/source/SpoolDirectorySource.java b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/source/SpoolDirectorySource.java
index 09f2b3c..9815e97 100644
--- a/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/source/SpoolDirectorySource.java
+++ b/flume-rpc-avro/src/main/java/org/apache/flume/rpc/avro/source/SpoolDirectorySource.java
@@ -1,320 +1,303 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
-
package org.apache.flume.rpc.avro.source;
+import static org.apache.flume.source.SpoolDirectorySourceConfigurationConstants.*;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
-import org.apache.flume.exception.ChannelException;
-import org.apache.flume.exception.ChannelFullException;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDrivenSource;
import org.apache.flume.FlumeException;
-import org.apache.flume.rpc.avro.serialization.ReliableSpoolingFileEventReader;
import org.apache.flume.conf.BatchSizeSupported;
import org.apache.flume.conf.Configurable;
+import org.apache.flume.exception.ChannelException;
+import org.apache.flume.exception.ChannelFullException;
import org.apache.flume.instrumentation.SourceCounter;
+import org.apache.flume.rpc.avro.serialization.ReliableSpoolingFileEventReader;
import org.apache.flume.serialization.DecodeErrorPolicy;
import org.apache.flume.serialization.LineDeserializer;
import org.apache.flume.source.AbstractSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.Locale;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
+public class SpoolDirectorySource extends AbstractSource
+ implements Configurable, EventDrivenSource, BatchSizeSupported {
+
+ private static final Logger logger = LoggerFactory.getLogger(SpoolDirectorySource.class);
+
+ /* Config options */
+ private String completedSuffix;
+ private String spoolDirectory;
+ private boolean fileHeader;
+ private String fileHeaderKey;
+ private boolean basenameHeader;
+ private String basenameHeaderKey;
+ private int batchSize;
+ private String includePattern;
+ private String ignorePattern;
+ private String trackerDirPath;
+ private String deserializerType;
+ private Context deserializerContext;
+ private String deletePolicy;
+ private String inputCharset;
+ private DecodeErrorPolicy decodeErrorPolicy;
+ private volatile boolean hasFatalError = false;
-import static org.apache.flume.source.SpoolDirectorySourceConfigurationConstants.*;
+ private SourceCounter sourceCounter;
+ ReliableSpoolingFileEventReader reader;
+ private ScheduledExecutorService executor;
+ private boolean backoff = true;
+ private boolean hitChannelException = false;
+ private boolean hitChannelFullException = false;
+ private int maxBackoff;
+ private ConsumeOrder consumeOrder;
+ private int pollDelay;
+ private boolean recursiveDirectorySearch;
+ private String trackingPolicy;
-public class SpoolDirectorySource extends AbstractSource
- implements Configurable, EventDrivenSource, BatchSizeSupported {
-
- private static final Logger logger = LoggerFactory.getLogger(SpoolDirectorySource.class);
-
- /* Config options */
- private String completedSuffix;
- private String spoolDirectory;
- private boolean fileHeader;
- private String fileHeaderKey;
- private boolean basenameHeader;
- private String basenameHeaderKey;
- private int batchSize;
- private String includePattern;
- private String ignorePattern;
- private String trackerDirPath;
- private String deserializerType;
- private Context deserializerContext;
- private String deletePolicy;
- private String inputCharset;
- private DecodeErrorPolicy decodeErrorPolicy;
- private volatile boolean hasFatalError = false;
-
- private SourceCounter sourceCounter;
- ReliableSpoolingFileEventReader reader;
- private ScheduledExecutorService executor;
- private boolean backoff = true;
- private boolean hitChannelException = false;
- private boolean hitChannelFullException = false;
- private int maxBackoff;
- private ConsumeOrder consumeOrder;
- private int pollDelay;
- private boolean recursiveDirectorySearch;
- private String trackingPolicy;
-
- @Override
- public synchronized void start() {
- logger.info("SpoolDirectorySource source starting with directory: {}",
- spoolDirectory);
-
- executor = Executors.newSingleThreadScheduledExecutor();
-
- File directory = new File(spoolDirectory);
- try {
- reader = new ReliableSpoolingFileEventReader.Builder()
- .spoolDirectory(directory)
- .completedSuffix(completedSuffix)
- .includePattern(includePattern)
- .ignorePattern(ignorePattern)
- .trackerDirPath(trackerDirPath)
- .annotateFileName(fileHeader)
- .fileNameHeader(fileHeaderKey)
- .annotateBaseName(basenameHeader)
- .baseNameHeader(basenameHeaderKey)
- .deserializerType(deserializerType)
- .deserializerContext(deserializerContext)
- .deletePolicy(deletePolicy)
- .inputCharset(inputCharset)
- .decodeErrorPolicy(decodeErrorPolicy)
- .consumeOrder(consumeOrder)
- .recursiveDirectorySearch(recursiveDirectorySearch)
- .trackingPolicy(trackingPolicy)
- .sourceCounter(sourceCounter)
- .build();
- } catch (IOException ioe) {
- throw new FlumeException("Error instantiating spooling event parser",
- ioe);
+ @Override
+ public synchronized void start() {
+ logger.info("SpoolDirectorySource source starting with directory: {}", spoolDirectory);
+
+ executor = Executors.newSingleThreadScheduledExecutor();
+
+ File directory = new File(spoolDirectory);
+ try {
+ reader = new ReliableSpoolingFileEventReader.Builder()
+ .spoolDirectory(directory)
+ .completedSuffix(completedSuffix)
+ .includePattern(includePattern)
+ .ignorePattern(ignorePattern)
+ .trackerDirPath(trackerDirPath)
+ .annotateFileName(fileHeader)
+ .fileNameHeader(fileHeaderKey)
+ .annotateBaseName(basenameHeader)
+ .baseNameHeader(basenameHeaderKey)
+ .deserializerType(deserializerType)
+ .deserializerContext(deserializerContext)
+ .deletePolicy(deletePolicy)
+ .inputCharset(inputCharset)
+ .decodeErrorPolicy(decodeErrorPolicy)
+ .consumeOrder(consumeOrder)
+ .recursiveDirectorySearch(recursiveDirectorySearch)
+ .trackingPolicy(trackingPolicy)
+ .sourceCounter(sourceCounter)
+ .build();
+ } catch (IOException ioe) {
+ throw new FlumeException("Error instantiating spooling event parser", ioe);
+ }
+
+ Runnable runner = new SpoolDirectoryRunnable(reader, sourceCounter);
+ executor.scheduleWithFixedDelay(runner, 0, pollDelay, TimeUnit.MILLISECONDS);
+
+ super.start();
+ logger.debug("SpoolDirectorySource source started");
+ sourceCounter.start();
+ }
+
+ @Override
+ public synchronized void stop() {
+ executor.shutdown();
+ try {
+ executor.awaitTermination(10L, TimeUnit.SECONDS);
+ } catch (InterruptedException ex) {
+ logger.info("Interrupted while awaiting termination", ex);
+ }
+ executor.shutdownNow();
+
+ super.stop();
+ sourceCounter.stop();
+ logger.info("SpoolDir source {} stopped. Metrics: {}", getName(), sourceCounter);
}
- Runnable runner = new SpoolDirectoryRunnable(reader, sourceCounter);
- executor.scheduleWithFixedDelay(
- runner, 0, pollDelay, TimeUnit.MILLISECONDS);
-
- super.start();
- logger.debug("SpoolDirectorySource source started");
- sourceCounter.start();
- }
-
- @Override
- public synchronized void stop() {
- executor.shutdown();
- try {
- executor.awaitTermination(10L, TimeUnit.SECONDS);
- } catch (InterruptedException ex) {
- logger.info("Interrupted while awaiting termination", ex);
+ @Override
+ public String toString() {
+ return "Spool Directory source " + getName() + ": { spoolDir: " + spoolDirectory + " }";
}
- executor.shutdownNow();
-
- super.stop();
- sourceCounter.stop();
- logger.info("SpoolDir source {} stopped. Metrics: {}", getName(), sourceCounter);
- }
-
- @Override
- public String toString() {
- return "Spool Directory source " + getName() +
- ": { spoolDir: " + spoolDirectory + " }";
- }
-
- @Override
- public synchronized void configure(Context context) {
- spoolDirectory = context.getString(SPOOL_DIRECTORY);
- Preconditions.checkState(spoolDirectory != null,
- "Configuration must specify a spooling directory");
-
- completedSuffix = context.getString(SPOOLED_FILE_SUFFIX,
- DEFAULT_SPOOLED_FILE_SUFFIX);
- deletePolicy = context.getString(DELETE_POLICY, DEFAULT_DELETE_POLICY);
- fileHeader = context.getBoolean(FILENAME_HEADER,
- DEFAULT_FILE_HEADER);
- fileHeaderKey = context.getString(FILENAME_HEADER_KEY,
- DEFAULT_FILENAME_HEADER_KEY);
- basenameHeader = context.getBoolean(BASENAME_HEADER,
- DEFAULT_BASENAME_HEADER);
- basenameHeaderKey = context.getString(BASENAME_HEADER_KEY,
- DEFAULT_BASENAME_HEADER_KEY);
- batchSize = context.getInteger(BATCH_SIZE,
- DEFAULT_BATCH_SIZE);
- inputCharset = context.getString(INPUT_CHARSET, DEFAULT_INPUT_CHARSET);
- decodeErrorPolicy = DecodeErrorPolicy.valueOf(
- context.getString(DECODE_ERROR_POLICY, DEFAULT_DECODE_ERROR_POLICY)
- .toUpperCase(Locale.ENGLISH));
-
- includePattern = context.getString(INCLUDE_PAT, DEFAULT_INCLUDE_PAT);
- ignorePattern = context.getString(IGNORE_PAT, DEFAULT_IGNORE_PAT);
- trackerDirPath = context.getString(TRACKER_DIR, DEFAULT_TRACKER_DIR);
-
- deserializerType = context.getString(DESERIALIZER, DEFAULT_DESERIALIZER);
- deserializerContext = new Context(context.getSubProperties(DESERIALIZER +
- "."));
-
- consumeOrder = ConsumeOrder.valueOf(context.getString(CONSUME_ORDER,
- DEFAULT_CONSUME_ORDER.toString()).toUpperCase(Locale.ENGLISH));
-
- pollDelay = context.getInteger(POLL_DELAY, DEFAULT_POLL_DELAY);
-
- recursiveDirectorySearch = context.getBoolean(RECURSIVE_DIRECTORY_SEARCH,
- DEFAULT_RECURSIVE_DIRECTORY_SEARCH);
-
- // "Hack" to support backwards compatibility with previous generation of
- // spooling directory source, which did not support deserializers
- Integer bufferMaxLineLength = context.getInteger(BUFFER_MAX_LINE_LENGTH);
- if (bufferMaxLineLength != null && deserializerType != null &&
- deserializerType.equalsIgnoreCase(DEFAULT_DESERIALIZER)) {
- deserializerContext.put(LineDeserializer.MAXLINE_KEY,
- bufferMaxLineLength.toString());
+
+ @Override
+ public synchronized void configure(Context context) {
+ spoolDirectory = context.getString(SPOOL_DIRECTORY);
+ Preconditions.checkState(spoolDirectory != null, "Configuration must specify a spooling directory");
+
+ completedSuffix = context.getString(SPOOLED_FILE_SUFFIX, DEFAULT_SPOOLED_FILE_SUFFIX);
+ deletePolicy = context.getString(DELETE_POLICY, DEFAULT_DELETE_POLICY);
+ fileHeader = context.getBoolean(FILENAME_HEADER, DEFAULT_FILE_HEADER);
+ fileHeaderKey = context.getString(FILENAME_HEADER_KEY, DEFAULT_FILENAME_HEADER_KEY);
+ basenameHeader = context.getBoolean(BASENAME_HEADER, DEFAULT_BASENAME_HEADER);
+ basenameHeaderKey = context.getString(BASENAME_HEADER_KEY, DEFAULT_BASENAME_HEADER_KEY);
+ batchSize = context.getInteger(BATCH_SIZE, DEFAULT_BATCH_SIZE);
+ inputCharset = context.getString(INPUT_CHARSET, DEFAULT_INPUT_CHARSET);
+ decodeErrorPolicy =
+ DecodeErrorPolicy.valueOf(context.getString(DECODE_ERROR_POLICY, DEFAULT_DECODE_ERROR_POLICY)
+ .toUpperCase(Locale.ENGLISH));
+
+ includePattern = context.getString(INCLUDE_PAT, DEFAULT_INCLUDE_PAT);
+ ignorePattern = context.getString(IGNORE_PAT, DEFAULT_IGNORE_PAT);
+ trackerDirPath = context.getString(TRACKER_DIR, DEFAULT_TRACKER_DIR);
+
+ deserializerType = context.getString(DESERIALIZER, DEFAULT_DESERIALIZER);
+ deserializerContext = new Context(context.getSubProperties(DESERIALIZER + "."));
+
+ consumeOrder = ConsumeOrder.valueOf(context.getString(CONSUME_ORDER, DEFAULT_CONSUME_ORDER.toString())
+ .toUpperCase(Locale.ENGLISH));
+
+ pollDelay = context.getInteger(POLL_DELAY, DEFAULT_POLL_DELAY);
+
+ recursiveDirectorySearch = context.getBoolean(RECURSIVE_DIRECTORY_SEARCH, DEFAULT_RECURSIVE_DIRECTORY_SEARCH);
+
+ // "Hack" to support backwards compatibility with previous generation of
+ // spooling directory source, which did not support deserializers
+ Integer bufferMaxLineLength = context.getInteger(BUFFER_MAX_LINE_LENGTH);
+ if (bufferMaxLineLength != null
+ && deserializerType != null
+ && deserializerType.equalsIgnoreCase(DEFAULT_DESERIALIZER)) {
+ deserializerContext.put(LineDeserializer.MAXLINE_KEY, bufferMaxLineLength.toString());
+ }
+
+ maxBackoff = context.getInteger(MAX_BACKOFF, DEFAULT_MAX_BACKOFF);
+ if (sourceCounter == null) {
+ sourceCounter = new SourceCounter(getName());
+ }
+ trackingPolicy = context.getString(TRACKING_POLICY, DEFAULT_TRACKING_POLICY);
}
- maxBackoff = context.getInteger(MAX_BACKOFF, DEFAULT_MAX_BACKOFF);
- if (sourceCounter == null) {
- sourceCounter = new SourceCounter(getName());
+ @VisibleForTesting
+ protected boolean hasFatalError() {
+ return hasFatalError;
}
- trackingPolicy = context.getString(TRACKING_POLICY, DEFAULT_TRACKING_POLICY);
- }
-
- @VisibleForTesting
- protected boolean hasFatalError() {
- return hasFatalError;
- }
-
-
- /**
- * The class always backs off, this exists only so that we can test without
- * taking a really long time.
- *
- * @param backoff - whether the source should backoff if the channel is full
- */
- @VisibleForTesting
- protected void setBackOff(boolean backoff) {
- this.backoff = backoff;
- }
-
- @VisibleForTesting
- protected boolean didHitChannelException() {
- return hitChannelException;
- }
-
- @VisibleForTesting
- protected boolean didHitChannelFullException() {
- return hitChannelFullException;
- }
-
- @VisibleForTesting
- protected SourceCounter getSourceCounter() {
- return sourceCounter;
- }
-
- @VisibleForTesting
- protected boolean getRecursiveDirectorySearch() {
- return recursiveDirectorySearch;
- }
-
- @Override
- public long getBatchSize() {
- return batchSize;
- }
-
- @VisibleForTesting
- protected class SpoolDirectoryRunnable implements Runnable {
- private ReliableSpoolingFileEventReader reader;
- private SourceCounter sourceCounter;
- public SpoolDirectoryRunnable(ReliableSpoolingFileEventReader reader,
- SourceCounter sourceCounter) {
- this.reader = reader;
- this.sourceCounter = sourceCounter;
+ /**
+ * The class always backs off, this exists only so that we can test without
+ * taking a really long time.
+ *
+ * @param backoff - whether the source should backoff if the channel is full
+ */
+ @VisibleForTesting
+ protected void setBackOff(boolean backoff) {
+ this.backoff = backoff;
+ }
+
+ @VisibleForTesting
+ protected boolean didHitChannelException() {
+ return hitChannelException;
+ }
+
+ @VisibleForTesting
+ protected boolean didHitChannelFullException() {
+ return hitChannelFullException;
+ }
+
+ @VisibleForTesting
+ protected SourceCounter getSourceCounter() {
+ return sourceCounter;
+ }
+
+ @VisibleForTesting
+ protected boolean getRecursiveDirectorySearch() {
+ return recursiveDirectorySearch;
}
@Override
- public void run() {
- int backoffInterval = 250;
- boolean readingEvents = false;
- try {
- while (!Thread.interrupted()) {
- readingEvents = true;
- List<Event> events = reader.readEvents(batchSize);
- readingEvents = false;
- if (events.isEmpty()) {
- break;
- }
- sourceCounter.addToEventReceivedCount(events.size());
- sourceCounter.incrementAppendBatchReceivedCount();
-
- try {
- getChannelProcessor().processEventBatch(events);
- reader.commit();
- } catch (ChannelFullException ex) {
- logger.warn("The channel is full, and cannot write data now. The " +
- "source will try again after " + backoffInterval +
- " milliseconds");
- sourceCounter.incrementChannelWriteFail();
- hitChannelFullException = true;
- backoffInterval = waitAndGetNewBackoffInterval(backoffInterval);
- continue;
- } catch (ChannelException ex) {
- logger.warn("The channel threw an exception, and cannot write data now. The " +
- "source will try again after " + backoffInterval +
- " milliseconds");
- sourceCounter.incrementChannelWriteFail();
- hitChannelException = true;
- backoffInterval = waitAndGetNewBackoffInterval(backoffInterval);
- continue;
- }
- backoffInterval = 250;
- sourceCounter.addToEventAcceptedCount(events.size());
- sourceCounter.incrementAppendBatchAcceptedCount();
+ public long getBatchSize() {
+ return batchSize;
+ }
+
+ @VisibleForTesting
+ protected class SpoolDirectoryRunnable implements Runnable {
+ private ReliableSpoolingFileEventReader reader;
+ private SourceCounter sourceCounter;
+
+ public SpoolDirectoryRunnable(ReliableSpoolingFileEventReader reader, SourceCounter sourceCounter) {
+ this.reader = reader;
+ this.sourceCounter = sourceCounter;
}
- } catch (Throwable t) {
- logger.error("FATAL: " + SpoolDirectorySource.this.toString() + ": " +
- "Uncaught exception in SpoolDirectorySource thread. " +
- "Restart or reconfigure Flume to continue processing.", t);
- if (readingEvents) {
- sourceCounter.incrementEventReadFail();
- } else {
- sourceCounter.incrementGenericProcessingFail();
+
+ @Override
+ public void run() {
+ int backoffInterval = 250;
+ boolean readingEvents = false;
+ try {
+ while (!Thread.interrupted()) {
+ readingEvents = true;
+ List<Event> events = reader.readEvents(batchSize);
+ readingEvents = false;
+ if (events.isEmpty()) {
+ break;
+ }
+ sourceCounter.addToEventReceivedCount(events.size());
+ sourceCounter.incrementAppendBatchReceivedCount();
+
+ try {
+ getChannelProcessor().processEventBatch(events);
+ reader.commit();
+ } catch (ChannelFullException ex) {
+ logger.warn("The channel is full, and cannot write data now. The "
+ + "source will try again after " + backoffInterval + " milliseconds");
+ sourceCounter.incrementChannelWriteFail();
+ hitChannelFullException = true;
+ backoffInterval = waitAndGetNewBackoffInterval(backoffInterval);
+ continue;
+ } catch (ChannelException ex) {
+ logger.warn("The channel threw an exception, and cannot write data now. The "
+ + "source will try again after "
+ + backoffInterval + " milliseconds");
+ sourceCounter.incrementChannelWriteFail();
+ hitChannelException = true;
+ backoffInterval = waitAndGetNewBackoffInterval(backoffInterval);
+ continue;
+ }
+ backoffInterval = 250;
+ sourceCounter.addToEventAcceptedCount(events.size());
+ sourceCounter.incrementAppendBatchAcceptedCount();
+ }
+ } catch (Throwable t) {
+ logger.error(
+ "FATAL: " + SpoolDirectorySource.this.toString() + ": "
+ + "Uncaught exception in SpoolDirectorySource thread. "
+ + "Restart or reconfigure Flume to continue processing.",
+ t);
+ if (readingEvents) {
+ sourceCounter.incrementEventReadFail();
+ } else {
+ sourceCounter.incrementGenericProcessingFail();
+ }
+ hasFatalError = true;
+ Throwables.propagate(t);
+ }
}
- hasFatalError = true;
- Throwables.propagate(t);
- }
- }
- private int waitAndGetNewBackoffInterval(int backoffInterval) throws InterruptedException {
- if (backoff) {
- TimeUnit.MILLISECONDS.sleep(backoffInterval);
- backoffInterval = backoffInterval << 1;
- backoffInterval = backoffInterval >= maxBackoff ? maxBackoff :
- backoffInterval;
- }
- return backoffInterval;
+ private int waitAndGetNewBackoffInterval(int backoffInterval) throws InterruptedException {
+ if (backoff) {
+ TimeUnit.MILLISECONDS.sleep(backoffInterval);
+ backoffInterval = backoffInterval << 1;
+ backoffInterval = backoffInterval >= maxBackoff ? maxBackoff : backoffInterval;
+ }
+ return backoffInterval;
+ }
}
- }
}
diff --git a/flume-rpc-avro/src/test/java/org/apache/flume/rpc/api/TestFailoverRpcClient.java b/flume-rpc-avro/src/test/java/org/apache/flume/rpc/api/TestFailoverRpcClient.java
index 593cb42..6f83302 100644
--- a/flume-rpc-avro/src/test/java/org/apache/flume/rpc/api/TestFailoverRpcClient.java
+++ b/flume-rpc-avro/src/test/java/org/apache/flume/rpc/api/TestFailoverRpcClient.java
@@ -1,20 +1,18 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.rpc.api;
@@ -23,150 +21,138 @@ import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
-
import org.apache.avro.ipc.Server;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.FlumeException;
import org.apache.flume.api.FailoverRpcClient;
import org.apache.flume.api.RpcClientFactory;
+import org.apache.flume.event.EventBuilder;
import org.apache.flume.rpc.avro.RpcTestUtils;
import org.apache.flume.rpc.avro.RpcTestUtils.OKAvroHandler;
-import org.apache.flume.event.EventBuilder;
import org.junit.Assert;
import org.junit.Test;
public class TestFailoverRpcClient {
- /**
- * Test a bunch of servers closing the one we are writing to and bringing
- * another one back online.
- *
- * @throws FlumeException
- * @throws EventDeliveryException
- * @throws InterruptedException
- */
-
- @Test
- public void testFailover()
- throws FlumeException, EventDeliveryException,InterruptedException {
- FailoverRpcClient client = null;
- Server server1 = RpcTestUtils.startServer(new OKAvroHandler());
- Server server2 = RpcTestUtils.startServer(new OKAvroHandler());
- Server server3 = RpcTestUtils.startServer(new OKAvroHandler());
- Properties props = new Properties();
- int s1Port = server1.getPort();
- int s2Port = server2.getPort();
- int s3Port = server3.getPort();
- props.put("client.type", "default_failover");
- props.put("hosts", "host1 host2 host3");
- props.put("hosts.host1", "127.0.0.1:" + String.valueOf(s1Port));
- props.put("hosts.host2", "127.0.0.1:" + String.valueOf(s2Port));
- props.put("hosts.host3", "127.0.0.1:" + String.valueOf(s3Port));
- client = (FailoverRpcClient) RpcClientFactory.getInstance(props);
- List<Event> events = new ArrayList<Event>();
- for (int i = 0; i < 50; i++) {
- events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
- }
- client.appendBatch(events);
- Assert.assertEquals(client.getLastConnectedServerAddress(),
- new InetSocketAddress("127.0.0.1", server1.getPort()));
- server1.close();
- Thread.sleep(1000L); // wait a second for the close to occur
- events = new ArrayList<Event>();
- for (int i = 0; i < 50; i++) {
- events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
- }
- client.appendBatch(events);
- Assert.assertEquals(new InetSocketAddress("localhost", server2.getPort()),
- client.getLastConnectedServerAddress());
- server2.close();
- Thread.sleep(1000L); // wait a second for the close to occur
- client.append(EventBuilder.withBody("Had a sandwich?",
- Charset.forName("UTF8")));
- Assert.assertEquals(new InetSocketAddress("localhost", server3.getPort()),
- client.getLastConnectedServerAddress());
- // Bring server 2 back.
- Server server4 = RpcTestUtils.startServer(new OKAvroHandler(), s2Port);
- server3.close();
- Thread.sleep(1000L); // wait a second for the close to occur
- events = new ArrayList<Event>();
- for (int i = 0; i < 50; i++) {
- events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
- }
- client.appendBatch(events);
- Assert.assertEquals(new InetSocketAddress("localhost", s2Port),
- client.getLastConnectedServerAddress());
+ /**
+ * Test a bunch of servers closing the one we are writing to and bringing
+ * another one back online.
+ *
+ * @throws FlumeException
+ * @throws EventDeliveryException
+ * @throws InterruptedException
+ */
+ @Test
+ public void testFailover() throws FlumeException, EventDeliveryException, InterruptedException {
+ FailoverRpcClient client = null;
+ Server server1 = RpcTestUtils.startServer(new OKAvroHandler());
+ Server server2 = RpcTestUtils.startServer(new OKAvroHandler());
+ Server server3 = RpcTestUtils.startServer(new OKAvroHandler());
+ Properties props = new Properties();
+ int s1Port = server1.getPort();
+ int s2Port = server2.getPort();
+ int s3Port = server3.getPort();
+ props.put("client.type", "default_failover");
+ props.put("hosts", "host1 host2 host3");
+ props.put("hosts.host1", "127.0.0.1:" + String.valueOf(s1Port));
+ props.put("hosts.host2", "127.0.0.1:" + String.valueOf(s2Port));
+ props.put("hosts.host3", "127.0.0.1:" + String.valueOf(s3Port));
+ client = (FailoverRpcClient) RpcClientFactory.getInstance(props);
+ List<Event> events = new ArrayList<Event>();
+ for (int i = 0; i < 50; i++) {
+ events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
+ }
+ client.appendBatch(events);
+ Assert.assertEquals(
+ client.getLastConnectedServerAddress(), new InetSocketAddress("127.0.0.1", server1.getPort()));
+ server1.close();
+ Thread.sleep(1000L); // wait a second for the close to occur
+ events = new ArrayList<Event>();
+ for (int i = 0; i < 50; i++) {
+ events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
+ }
+ client.appendBatch(events);
+ Assert.assertEquals(
+ new InetSocketAddress("localhost", server2.getPort()), client.getLastConnectedServerAddress());
+ server2.close();
+ Thread.sleep(1000L); // wait a second for the close to occur
+ client.append(EventBuilder.withBody("Had a sandwich?", Charset.forName("UTF8")));
+ Assert.assertEquals(
+ new InetSocketAddress("localhost", server3.getPort()), client.getLastConnectedServerAddress());
+ // Bring server 2 back.
+ Server server4 = RpcTestUtils.startServer(new OKAvroHandler(), s2Port);
+ server3.close();
+ Thread.sleep(1000L); // wait a second for the close to occur
+ events = new ArrayList<Event>();
+ for (int i = 0; i < 50; i++) {
+ events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
+ }
+ client.appendBatch(events);
+ Assert.assertEquals(new InetSocketAddress("localhost", s2Port), client.getLastConnectedServerAddress());
- Server server5 = RpcTestUtils.startServer(new OKAvroHandler(), s1Port);
- // Make sure we are still talking to server 4
+ Server server5 = RpcTestUtils.startServer(new OKAvroHandler(), s1Port);
+ // Make sure we are still talking to server 4
- client
- .append(EventBuilder.withBody("Had a mango?", Charset.forName("UTF8")));
- Assert.assertEquals(new InetSocketAddress("localhost", s2Port),
- client.getLastConnectedServerAddress());
- server4.close();
- Thread.sleep(1000L); // wait a second for the close to occur
- events = new ArrayList<Event>();
- for (int i = 0; i < 50; i++) {
- events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
- }
- client.appendBatch(events);
- Assert.assertEquals(new InetSocketAddress("localhost", s1Port),
- client.getLastConnectedServerAddress());
- server5.close();
- Thread.sleep(1000L); // wait a second for the close to occur
- Server server6 = RpcTestUtils.startServer(new OKAvroHandler(), s1Port);
- client.append(EventBuilder.withBody("Had a whole watermelon?", Charset.forName("UTF8")));
- Assert.assertEquals(new InetSocketAddress("localhost", s1Port),
- client.getLastConnectedServerAddress());
+ client.append(EventBuilder.withBody("Had a mango?", Charset.forName("UTF8")));
+ Assert.assertEquals(new InetSocketAddress("localhost", s2Port), client.getLastConnectedServerAddress());
+ server4.close();
+ Thread.sleep(1000L); // wait a second for the close to occur
+ events = new ArrayList<Event>();
+ for (int i = 0; i < 50; i++) {
+ events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
+ }
+ client.appendBatch(events);
+ Assert.assertEquals(new InetSocketAddress("localhost", s1Port), client.getLastConnectedServerAddress());
+ server5.close();
+ Thread.sleep(1000L); // wait a second for the close to occur
+ Server server6 = RpcTestUtils.startServer(new OKAvroHandler(), s1Port);
+ client.append(EventBuilder.withBody("Had a whole watermelon?", Charset.forName("UTF8")));
+ Assert.assertEquals(new InetSocketAddress("localhost", s1Port), client.getLastConnectedServerAddress());
- server6.close();
- Thread.sleep(1000L); // wait a second for the close to occur
- Server server7 = RpcTestUtils.startServer(new OKAvroHandler(), s3Port);
- events = new ArrayList<Event>();
- for (int i = 0; i < 50; i++) {
- events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
+ server6.close();
+ Thread.sleep(1000L); // wait a second for the close to occur
+ Server server7 = RpcTestUtils.startServer(new OKAvroHandler(), s3Port);
+ events = new ArrayList<Event>();
+ for (int i = 0; i < 50; i++) {
+ events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
+ }
+ client.appendBatch(events);
+ Assert.assertEquals(new InetSocketAddress("localhost", s3Port), client.getLastConnectedServerAddress());
+ server7.close();
}
- client.appendBatch(events);
- Assert.assertEquals(new InetSocketAddress("localhost", s3Port),
- client.getLastConnectedServerAddress());
- server7.close();
- }
- /**
- * Try writing to some servers and then kill them all.
- *
- * @throws FlumeException
- * @throws EventDeliveryException
- */
- @Test(
- expected = EventDeliveryException.class)
- public void testFailedServers() throws FlumeException, EventDeliveryException {
- FailoverRpcClient client = null;
- Server server1 = RpcTestUtils.startServer(new OKAvroHandler());
- Server server2 = RpcTestUtils.startServer(new OKAvroHandler());
- Server server3 = RpcTestUtils.startServer(new OKAvroHandler());
- Properties props = new Properties();
- props.put("client.type", "default_failover");
+ /**
+ * Try writing to some servers and then kill them all.
+ *
+ * @throws FlumeException
+ * @throws EventDeliveryException
+ */
+ @Test(expected = EventDeliveryException.class)
+ public void testFailedServers() throws FlumeException, EventDeliveryException {
+ FailoverRpcClient client = null;
+ Server server1 = RpcTestUtils.startServer(new OKAvroHandler());
+ Server server2 = RpcTestUtils.startServer(new OKAvroHandler());
+ Server server3 = RpcTestUtils.startServer(new OKAvroHandler());
+ Properties props = new Properties();
+ props.put("client.type", "default_failover");
- props.put("hosts", "host1 host2 host3");
- props.put("hosts.host1", "localhost:" + String.valueOf(server1.getPort()));
- props.put("hosts.host2", "localhost:" + String.valueOf(server2.getPort()));
- props.put("hosts.host3", " localhost:" + String.valueOf(server3.getPort()));
- client = (FailoverRpcClient) RpcClientFactory.getInstance(props);
- List<Event> events = new ArrayList<Event>();
- for (int i = 0; i < 50; i++) {
- events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
+ props.put("hosts", "host1 host2 host3");
+ props.put("hosts.host1", "localhost:" + String.valueOf(server1.getPort()));
+ props.put("hosts.host2", "localhost:" + String.valueOf(server2.getPort()));
+ props.put("hosts.host3", " localhost:" + String.valueOf(server3.getPort()));
+ client = (FailoverRpcClient) RpcClientFactory.getInstance(props);
+ List<Event> events = new ArrayList<Event>();
+ for (int i = 0; i < 50; i++) {
+ events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
+ }
+ client.appendBatch(events);
+ server1.close();
+ server2.close();
+ server3.close();
+ events = new ArrayList<Event>();
+ for (int i = 0; i < 50; i++) {
+ events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
+ }
+ client.appendBatch(events);
}
- client.appendBatch(events);
- server1.close();
- server2.close();
- server3.close();
- events = new ArrayList<Event>();
- for (int i = 0; i < 50; i++) {
- events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
- }
- client.appendBatch(events);
- }
-
}
diff --git a/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/RpcTestUtils.java b/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/RpcTestUtils.java
index 79bae34..e3b2724 100644
--- a/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/RpcTestUtils.java
+++ b/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/RpcTestUtils.java
@@ -1,29 +1,31 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.rpc.avro;
+import io.netty.channel.ChannelPipeline;
+import io.netty.handler.codec.compression.JZlibDecoder;
+import io.netty.handler.codec.compression.JZlibEncoder;
+import io.netty.handler.codec.compression.ZlibEncoder;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
-
+import junit.framework.Assert;
import org.apache.avro.ipc.Responder;
import org.apache.avro.ipc.Server;
import org.apache.avro.ipc.netty.NettyServer;
@@ -40,325 +42,324 @@ import org.apache.flume.source.avro.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import io.netty.channel.ChannelPipeline;
-import io.netty.handler.codec.compression.JZlibDecoder;
-import io.netty.handler.codec.compression.JZlibEncoder;
-import io.netty.handler.codec.compression.ZlibEncoder;
-import junit.framework.Assert;
-
/**
* Helpers for Netty Avro RPC testing
*/
public class RpcTestUtils {
- private static final Logger logger = LoggerFactory.getLogger(RpcTestUtils.class);
-
- private static final String localhost = "localhost";
-
- /**
- * Helper method for testing simple (single) appends on handlers
- * @param handler
- * @throws FlumeException
- * @throws EventDeliveryException
- */
- public static void handlerSimpleAppendTest(AvroSourceProtocol handler)
- throws FlumeException, EventDeliveryException {
- handlerSimpleAppendTest(handler, false, false, 0);
- }
-
- /**
- * Helper method for testing simple (single) with compression level 6 appends on handlers
- * @param handler
- * @throws FlumeException
- * @throws EventDeliveryException
- */
- public static void handlerSimpleAppendTest(AvroSourceProtocol handler,
- boolean enableServerCompression,
- boolean enableClientCompression, int compressionLevel)
- throws FlumeException, EventDeliveryException {
- NettyAvroRpcClient client = null;
- Server server = startServer(handler, 0, enableServerCompression);
- try {
- Properties starterProp = new Properties();
- if (enableClientCompression) {
- starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "deflate");
- starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_LEVEL,
- "" + compressionLevel);
- } else {
- starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "none");
- }
- client = getStockLocalClient(server.getPort(), starterProp);
- boolean isActive = client.isActive();
- Assert.assertTrue("Client should be active", isActive);
- client.append(EventBuilder.withBody("wheee!!!", Charset.forName("UTF8")));
- } finally {
- stopServer(server);
- if (client != null) client.close();
- }
- }
-
- public static void handlerBatchAppendTest(AvroSourceProtocol handler)
- throws FlumeException, EventDeliveryException {
- handlerBatchAppendTest(handler, false, false, 0);
- }
-
- /**
- * Helper method for testing batch appends on handlers
- * @param handler
- * @throws FlumeException
- * @throws EventDeliveryException
- */
- public static void handlerBatchAppendTest(AvroSourceProtocol handler,
- boolean enableServerCompression,
- boolean enableClientCompression, int compressionLevel)
- throws FlumeException, EventDeliveryException {
- NettyAvroRpcClient client = null;
- Server server = startServer(handler, 0 , enableServerCompression);
- try {
-
- Properties starterProp = new Properties();
- if (enableClientCompression) {
- starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "deflate");
- starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_LEVEL,
- "" + compressionLevel);
- } else {
- starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "none");
- }
-
- client = getStockLocalClient(server.getPort(), starterProp);
- boolean isActive = client.isActive();
- Assert.assertTrue("Client should be active", isActive);
-
- int batchSize = client.getBatchSize();
- List<Event> events = new ArrayList<Event>();
- for (int i = 0; i < batchSize; i++) {
- events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
- }
- client.appendBatch(events);
-
- } finally {
- stopServer(server);
- if (client != null) client.close();
- }
- }
-
- /**
- * Helper method for constructing a Netty RPC client that talks to localhost.
- */
- public static NettyAvroRpcClient getStockLocalClient(int port) {
- Properties props = new Properties();
-
- return getStockLocalClient(port, props);
- }
-
- public static NettyAvroRpcClient getStockLocalClient(int port, Properties starterProp) {
- starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS, "h1");
- starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + "h1",
- "127.0.0.1" + ":" + port);
- NettyAvroRpcClient client = NettyAvroRpcClient.newInstance();
- client.configure(starterProp);
-
- return client;
- }
-
- /**
- * Start a NettyServer, wait a moment for it to spin up, and return it.
- */
- public static Server startServer(AvroSourceProtocol handler, int port, boolean enableCompression) {
- Responder responder = new SpecificResponder(AvroSourceProtocol.class, handler);
- Server server = null;
- try {
- if (enableCompression) {
- server = new NettyServer(responder, new InetSocketAddress(localhost, port),
- (ch) -> {
- ChannelPipeline pipeline = ch.pipeline();
- ZlibEncoder encoder = new JZlibEncoder(6);
- pipeline.addFirst("deflater", encoder);
- pipeline.addFirst("inflater", new JZlibDecoder());
- });
- } else {
- server = new NettyServer(responder, new InetSocketAddress(localhost, port));
- }
- server.start();
- logger.info("Server started on hostname: {}, port: {}",
- new Object[]{localhost, Integer.toString(server.getPort())});
- Thread.sleep(300L);
- } catch (InterruptedException ex) {
- logger.error("Thread interrupted. Exception follows.", ex);
- Thread.currentThread().interrupt();
- }
-
- return server;
- }
-
- public static Server startServer(AvroSourceProtocol handler) {
- return startServer(handler, 0, false);
- }
-
- public static Server startServer(AvroSourceProtocol handler, int port) {
- return startServer(handler, port, false);
- }
-
-
- /**
- * Request that the specified Server stop, and attempt to wait for it to exit.
- * @param server A running NettyServer
- */
- public static void stopServer(Server server) {
- try {
- int port = server.getPort();
- server.close();
- server.join();
- logger.info("Server stopped on port: {}", port);
- } catch (InterruptedException ex) {
- logger.error("Thread interrupted. Exception follows.", ex);
- Thread.currentThread().interrupt();
- }
- }
-
- public static class LoadBalancedAvroHandler implements AvroSourceProtocol {
+ private static final Logger logger = LoggerFactory.getLogger(RpcTestUtils.class);
- private int appendCount = 0;
- private int appendBatchCount = 0;
+ private static final String localhost = "localhost";
- private boolean failed = false;
-
- public int getAppendCount() {
- return appendCount;
+ /**
+ * Helper method for testing simple (single) appends on handlers
+ * @param handler
+ * @throws FlumeException
+ * @throws EventDeliveryException
+ */
+ public static void handlerSimpleAppendTest(AvroSourceProtocol handler)
+ throws FlumeException, EventDeliveryException {
+ handlerSimpleAppendTest(handler, false, false, 0);
}
- public int getAppendBatchCount() {
- return appendBatchCount;
+ /**
+ * Helper method for testing simple (single) with compression level 6 appends on handlers
+ * @param handler
+ * @throws FlumeException
+ * @throws EventDeliveryException
+ */
+ public static void handlerSimpleAppendTest(
+ AvroSourceProtocol handler,
+ boolean enableServerCompression,
+ boolean enableClientCompression,
+ int compressionLevel)
+ throws FlumeException, EventDeliveryException {
+ NettyAvroRpcClient client = null;
+ Server server = startServer(handler, 0, enableServerCompression);
+ try {
+ Properties starterProp = new Properties();
+ if (enableClientCompression) {
+ starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "deflate");
+ starterProp.setProperty(
+ RpcClientConfigurationConstants.CONFIG_COMPRESSION_LEVEL, "" + compressionLevel);
+ } else {
+ starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "none");
+ }
+ client = getStockLocalClient(server.getPort(), starterProp);
+ boolean isActive = client.isActive();
+ Assert.assertTrue("Client should be active", isActive);
+ client.append(EventBuilder.withBody("wheee!!!", Charset.forName("UTF8")));
+ } finally {
+ stopServer(server);
+ if (client != null) client.close();
+ }
}
- public boolean isFailed() {
- return failed;
+ public static void handlerBatchAppendTest(AvroSourceProtocol handler)
+ throws FlumeException, EventDeliveryException {
+ handlerBatchAppendTest(handler, false, false, 0);
}
- public void setFailed() {
- this.failed = true;
+ /**
+ * Helper method for testing batch appends on handlers
+ * @param handler
+ * @throws FlumeException
+ * @throws EventDeliveryException
+ */
+ public static void handlerBatchAppendTest(
+ AvroSourceProtocol handler,
+ boolean enableServerCompression,
+ boolean enableClientCompression,
+ int compressionLevel)
+ throws FlumeException, EventDeliveryException {
+ NettyAvroRpcClient client = null;
+ Server server = startServer(handler, 0, enableServerCompression);
+ try {
+
+ Properties starterProp = new Properties();
+ if (enableClientCompression) {
+ starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "deflate");
+ starterProp.setProperty(
+ RpcClientConfigurationConstants.CONFIG_COMPRESSION_LEVEL, "" + compressionLevel);
+ } else {
+ starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "none");
+ }
+
+ client = getStockLocalClient(server.getPort(), starterProp);
+ boolean isActive = client.isActive();
+ Assert.assertTrue("Client should be active", isActive);
+
+ int batchSize = client.getBatchSize();
+ List<Event> events = new ArrayList<Event>();
+ for (int i = 0; i < batchSize; i++) {
+ events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
+ }
+ client.appendBatch(events);
+
+ } finally {
+ stopServer(server);
+ if (client != null) client.close();
+ }
}
- public void setOK() {
- this.failed = false;
- }
+ /**
+ * Helper method for constructing a Netty RPC client that talks to localhost.
+ */
+ public static NettyAvroRpcClient getStockLocalClient(int port) {
+ Properties props = new Properties();
- @Override
- public Status append(AvroFlumeEvent event) {
- if (failed) {
- logger.debug("Event rejected");
- return Status.FAILED;
- }
- logger.debug("LB: Received event from append(): {}",
- new String(event.getBody().array(), Charset.forName("UTF8")));
- appendCount++;
- return Status.OK;
+ return getStockLocalClient(port, props);
}
- @Override
- public Status appendBatch(List<AvroFlumeEvent> events) {
- if (failed) {
- logger.debug("Event batch rejected");
- return Status.FAILED;
- }
- logger.debug("LB: Received {} events from appendBatch()",
- events.size());
-
- appendBatchCount++;
- return Status.OK;
- }
- }
-
- /**
- * A service that logs receipt of the request and returns OK
- */
- public static class OKAvroHandler implements AvroSourceProtocol {
-
- @Override
- public Status append(AvroFlumeEvent event) {
- logger.info("OK: Received event from append(): {}",
- new String(event.getBody().array(), Charset.forName("UTF8")));
- return Status.OK;
- }
+ public static NettyAvroRpcClient getStockLocalClient(int port, Properties starterProp) {
+ starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS, "h1");
+ starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + "h1", "127.0.0.1" + ":" + port);
+ NettyAvroRpcClient client = NettyAvroRpcClient.newInstance();
+ client.configure(starterProp);
- @Override
- public Status appendBatch(List<AvroFlumeEvent> events) {
- logger.info("OK: Received {} events from appendBatch()", events.size());
- return Status.OK;
+ return client;
}
- }
-
- /**
- * A service that logs receipt of the request and returns Failed
- */
- public static class FailedAvroHandler implements AvroSourceProtocol {
-
- @Override
- public Status append(AvroFlumeEvent event) {
- logger.info("Failed: Received event from append(): {}",
- new String(event.getBody().array(), Charset.forName("UTF8")));
- return Status.FAILED;
+ /**
+ * Start a NettyServer, wait a moment for it to spin up, and return it.
+ */
+ public static Server startServer(AvroSourceProtocol handler, int port, boolean enableCompression) {
+ Responder responder = new SpecificResponder(AvroSourceProtocol.class, handler);
+ Server server = null;
+ try {
+ if (enableCompression) {
+ server = new NettyServer(responder, new InetSocketAddress(localhost, port), (ch) -> {
+ ChannelPipeline pipeline = ch.pipeline();
+ ZlibEncoder encoder = new JZlibEncoder(6);
+ pipeline.addFirst("deflater", encoder);
+ pipeline.addFirst("inflater", new JZlibDecoder());
+ });
+ } else {
+ server = new NettyServer(responder, new InetSocketAddress(localhost, port));
+ }
+ server.start();
+ logger.info(
+ "Server started on hostname: {}, port: {}",
+ new Object[] {localhost, Integer.toString(server.getPort())});
+ Thread.sleep(300L);
+ } catch (InterruptedException ex) {
+ logger.error("Thread interrupted. Exception follows.", ex);
+ Thread.currentThread().interrupt();
+ }
+
+ return server;
}
- @Override
- public Status appendBatch(List<AvroFlumeEvent> events) {
- logger.info("Failed: Received {} events from appendBatch()", events.size());
- return Status.FAILED;
+ public static Server startServer(AvroSourceProtocol handler) {
+ return startServer(handler, 0, false);
}
- }
-
- /**
- * A service that logs receipt of the request and returns Unknown
- */
- public static class UnknownAvroHandler implements AvroSourceProtocol {
+ public static Server startServer(AvroSourceProtocol handler, int port) {
+ return startServer(handler, port, false);
+ }
- @Override
- public Status append(AvroFlumeEvent event) {
- logger.info("Unknown: Received event from append(): {}",
- new String(event.getBody().array(), Charset.forName("UTF8")));
- return Status.UNKNOWN;
+ /**
+ * Request that the specified Server stop, and attempt to wait for it to exit.
+ * @param server A running NettyServer
+ */
+ public static void stopServer(Server server) {
+ try {
+ int port = server.getPort();
+ server.close();
+ server.join();
+ logger.info("Server stopped on port: {}", port);
+ } catch (InterruptedException ex) {
+ logger.error("Thread interrupted. Exception follows.", ex);
+ Thread.currentThread().interrupt();
+ }
}
- @Override
- public Status appendBatch(List<AvroFlumeEvent> events) {
- logger.info("Unknown: Received {} events from appendBatch()", events.size());
- return Status.UNKNOWN;
+ public static class LoadBalancedAvroHandler implements AvroSourceProtocol {
+
+ private int appendCount = 0;
+ private int appendBatchCount = 0;
+
+ private boolean failed = false;
+
+ public int getAppendCount() {
+ return appendCount;
+ }
+
+ public int getAppendBatchCount() {
+ return appendBatchCount;
+ }
+
+ public boolean isFailed() {
+ return failed;
+ }
+
+ public void setFailed() {
+ this.failed = true;
+ }
+
+ public void setOK() {
+ this.failed = false;
+ }
+
+ @Override
+ public Status append(AvroFlumeEvent event) {
+ if (failed) {
+ logger.debug("Event rejected");
+ return Status.FAILED;
+ }
+ logger.debug(
+ "LB: Received event from append(): {}",
+ new String(event.getBody().array(), Charset.forName("UTF8")));
+ appendCount++;
+ return Status.OK;
+ }
+
+ @Override
+ public Status appendBatch(List<AvroFlumeEvent> events) {
+ if (failed) {
+ logger.debug("Event batch rejected");
+ return Status.FAILED;
+ }
+ logger.debug("LB: Received {} events from appendBatch()", events.size());
+
+ appendBatchCount++;
+ return Status.OK;
+ }
}
- }
-
- /**
- * A service that logs receipt of the request and then throws an exception
- */
- public static class ThrowingAvroHandler implements AvroSourceProtocol.Callback {
-
- @Override
- public void append(AvroFlumeEvent event, org.apache.avro.ipc.Callback<Status> callback)
- throws java.io.IOException {
- logger.info("Throwing: Received event from append(): {}",
- new String(event.getBody().array(), Charset.forName("UTF8")));
- throw new java.io.IOException("Handler smash!");
+
+ /**
+ * A service that logs receipt of the request and returns OK
+ */
+ public static class OKAvroHandler implements AvroSourceProtocol {
+
+ @Override
+ public Status append(AvroFlumeEvent event) {
+ logger.info(
+ "OK: Received event from append(): {}",
+ new String(event.getBody().array(), Charset.forName("UTF8")));
+ return Status.OK;
+ }
+
+ @Override
+ public Status appendBatch(List<AvroFlumeEvent> events) {
+ logger.info("OK: Received {} events from appendBatch()", events.size());
+ return Status.OK;
+ }
}
- @Override
- public Status append(AvroFlumeEvent event) {
- logger.info("Throwing unavailable: Received event from append(): {}",
- new String(event.getBody().array(), Charset.forName("UTF8")));
- return null;
+ /**
+ * A service that logs receipt of the request and returns Failed
+ */
+ public static class FailedAvroHandler implements AvroSourceProtocol {
+
+ @Override
+ public Status append(AvroFlumeEvent event) {
+ logger.info(
+ "Failed: Received event from append(): {}",
+ new String(event.getBody().array(), Charset.forName("UTF8")));
+ return Status.FAILED;
+ }
+
+ @Override
+ public Status appendBatch(List<AvroFlumeEvent> events) {
+ logger.info("Failed: Received {} events from appendBatch()", events.size());
+ return Status.FAILED;
+ }
}
- @Override
- public void appendBatch(List<AvroFlumeEvent> events, org.apache.avro.ipc.Callback<Status> callback)
- throws java.io.IOException {
- logger.info("Throwing: Received {} events from appendBatch()", events.size());
- throw new java.io.IOException("Handler smash!");
+ /**
+ * A service that logs receipt of the request and returns Unknown
+ */
+ public static class UnknownAvroHandler implements AvroSourceProtocol {
+
+ @Override
+ public Status append(AvroFlumeEvent event) {
+ logger.info(
+ "Unknown: Received event from append(): {}",
+ new String(event.getBody().array(), Charset.forName("UTF8")));
+ return Status.UNKNOWN;
+ }
+
+ @Override
+ public Status appendBatch(List<AvroFlumeEvent> events) {
+ logger.info("Unknown: Received {} events from appendBatch()", events.size());
+ return Status.UNKNOWN;
+ }
}
- @Override
- public Status appendBatch(List<AvroFlumeEvent> events) {
- logger.info("Throwing unavailable: Received {} events from appendBatch()", events.size());
- return null;
+ /**
+ * A service that logs receipt of the request and then throws an exception
+ */
+ public static class ThrowingAvroHandler implements AvroSourceProtocol.Callback {
+
+ @Override
+ public void append(AvroFlumeEvent event, org.apache.avro.ipc.Callback<Status> callback)
+ throws java.io.IOException {
+ logger.info(
+ "Throwing: Received event from append(): {}",
+ new String(event.getBody().array(), Charset.forName("UTF8")));
+ throw new java.io.IOException("Handler smash!");
+ }
+
+ @Override
+ public Status append(AvroFlumeEvent event) {
+ logger.info(
+ "Throwing unavailable: Received event from append(): {}",
+ new String(event.getBody().array(), Charset.forName("UTF8")));
+ return null;
+ }
+
+ @Override
+ public void appendBatch(List<AvroFlumeEvent> events, org.apache.avro.ipc.Callback<Status> callback)
+ throws java.io.IOException {
+ logger.info("Throwing: Received {} events from appendBatch()", events.size());
+ throw new java.io.IOException("Handler smash!");
+ }
+
+ @Override
+ public Status appendBatch(List<AvroFlumeEvent> events) {
+ logger.info("Throwing unavailable: Received {} events from appendBatch()", events.size());
+ return null;
+ }
}
- }
}
diff --git a/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/TestLoadBalancingRpcClient.java b/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/TestLoadBalancingRpcClient.java
index 1bae722..11ae435 100644
--- a/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/TestLoadBalancingRpcClient.java
+++ b/flume-rpc-avro/src/test/java/org/apache/flume/rpc/avro/TestLoadBalancingRpcClient.java
@@ -1,23 +1,26 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.flume.rpc.avro;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
import junit.framework.Assert;
import org.apache.avro.ipc.Server;
import org.apache.flume.Event;
@@ -33,658 +36,643 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Properties;
-import java.util.Set;
-
public class TestLoadBalancingRpcClient {
- private static final Logger LOGGER = LoggerFactory.getLogger(TestLoadBalancingRpcClient.class);
-
- @Test(expected = FlumeException.class)
- public void testCreatingLbClientSingleHost() {
- Server server1 = null;
- RpcClient c = null;
- try {
- server1 = RpcTestUtils.startServer(new OKAvroHandler());
- Properties p = new Properties();
- p.put("host1", "127.0.0.1:" + server1.getPort());
- p.put("hosts", "host1");
- p.put("client.type", "default_loadbalance");
- RpcClientFactory.getInstance(p);
- } finally {
- if (server1 != null) server1.close();
- if (c != null) c.close();
- }
- }
-
- @Test
- public void testTwoHostFailover() throws Exception {
- Server s1 = null;
- Server s2 = null;
- RpcClient c = null;
- try {
- LoadBalancedAvroHandler h1 = new LoadBalancedAvroHandler();
- LoadBalancedAvroHandler h2 = new LoadBalancedAvroHandler();
-
- s1 = RpcTestUtils.startServer(h1);
- s2 = RpcTestUtils.startServer(h2);
-
- Properties p = new Properties();
- p.put("hosts", "h1 h2");
- p.put("client.type", "default_loadbalance");
- p.put("hosts.h1", "127.0.0.1:" + s1.getPort());
- p.put("hosts.h2", "127.0.0.1:" + s2.getPort());
-
- c = RpcClientFactory.getInstance(p);
- Assert.assertTrue(c instanceof LoadBalancingRpcClient);
-
- for (int i = 0; i < 100; i++) {
- if (i == 20) {
- h2.setFailed();
- } else if (i == 40) {
- h2.setOK();
+ private static final Logger LOGGER = LoggerFactory.getLogger(TestLoadBalancingRpcClient.class);
+
+ @Test(expected = FlumeException.class)
+ public void testCreatingLbClientSingleHost() {
+ Server server1 = null;
+ RpcClient c = null;
+ try {
+ server1 = RpcTestUtils.startServer(new OKAvroHandler());
+ Properties p = new Properties();
+ p.put("host1", "127.0.0.1:" + server1.getPort());
+ p.put("hosts", "host1");
+ p.put("client.type", "default_loadbalance");
+ RpcClientFactory.getInstance(p);
+ } finally {
+ if (server1 != null) server1.close();
+ if (c != null) c.close();
}
- c.append(getEvent(i));
- }
-
- Assert.assertEquals(60, h1.getAppendCount());
- Assert.assertEquals(40, h2.getAppendCount());
- } finally {
- if (s1 != null) s1.close();
- if (s2 != null) s2.close();
- if (c != null) c.close();
}
- }
-
- // This will fail without FLUME-1823
- @Test(expected = EventDeliveryException.class)
- public void testTwoHostFailoverThrowAfterClose() throws Exception {
- Server s1 = null;
- Server s2 = null;
- RpcClient c = null;
- try {
- LoadBalancedAvroHandler h1 = new LoadBalancedAvroHandler();
- LoadBalancedAvroHandler h2 = new LoadBalancedAvroHandler();
-
- s1 = RpcTestUtils.startServer(h1);
- s2 = RpcTestUtils.startServer(h2);
-
- Properties p = new Properties();
- p.put("hosts", "h1 h2");
- p.put("client.type", "default_loadbalance");
- p.put("hosts.h1", "127.0.0.1:" + s1.getPort());
- p.put("hosts.h2", "127.0.0.1:" + s2.getPort());
-
- c = RpcClientFactory.getInstance(p);
- Assert.assertTrue(c instanceof LoadBalancingRpcClient);
-
- for (int i = 0; i < 100; i++) {
- if (i == 20) {
- h2.setFailed();
- } else if (i == 40) {
- h2.setOK();
+
+ @Test
+ public void testTwoHostFailover() throws Exception {
+ Server s1 = null;
+ Server s2 = null;
+ RpcClient c = null;
+ try {
+ LoadBalancedAvroHandler h1 = new LoadBalancedAvroHandler();
+ LoadBalancedAvroHandler h2 = new LoadBalancedAvroHandler();
+
+ s1 = RpcTestUtils.startServer(h1);
+ s2 = RpcTestUtils.startServer(h2);
+
+ Properties p = new Properties();
+ p.put("hosts", "h1 h2");
+ p.put("client.type", "default_loadbalance");
+ p.put("hosts.h1", "127.0.0.1:" + s1.getPort());
+ p.put("hosts.h2", "127.0.0.1:" + s2.getPort());
+
+ c = RpcClientFactory.getInstance(p);
+ Assert.assertTrue(c instanceof LoadBalancingRpcClient);
+
+ for (int i = 0; i < 100; i++) {
+ if (i == 20) {
+ h2.setFailed();
+ } else if (i == 40) {
+ h2.setOK();
+ }
+ c.append(getEvent(i));
+ }
+
+ Assert.assertEquals(60, h1.getAppendCount());
+ Assert.assertEquals(40, h2.getAppendCount());
+ } finally {
+ if (s1 != null) s1.close();
+ if (s2 != null) s2.close();
+ if (c != null) c.close();
}
- c.append(getEvent(i));
- }
-
- Assert.assertEquals(60, h1.getAppendCount());
- Assert.assertEquals(40, h2.getAppendCount());
- if (c != null) c.close();
- c.append(getEvent(3));
- Assert.fail();
- } finally {
- if (s1 != null) s1.close();
- if (s2 != null) s2.close();
- }
- }
-
- /**
- * Ensure that we can tolerate a host that is completely down.
- *
- * @throws Exception
- */
- @Test
- public void testTwoHostsOneDead() throws Exception {
- LOGGER.info("Running testTwoHostsOneDead...");
- Server s1 = null;
- RpcClient c1 = null;
- RpcClient c2 = null;
- try {
- LoadBalancedAvroHandler h1 = new LoadBalancedAvroHandler();
- s1 = RpcTestUtils.startServer(h1);
- // do not create a 2nd server (assume it's "down")
-
- Properties p = new Properties();
- p.put("hosts", "h1 h2");
- p.put("client.type", "default_loadbalance");
- p.put("hosts.h1", "127.0.0.1:" + 0); // port 0 should always be closed
- p.put("hosts.h2", "127.0.0.1:" + s1.getPort());
-
- // test batch API
- c1 = RpcClientFactory.getInstance(p);
- Assert.assertTrue(c1 instanceof LoadBalancingRpcClient);
-
- for (int i = 0; i < 10; i++) {
- c1.appendBatch(getBatchedEvent(i));
- }
- Assert.assertEquals(10, h1.getAppendBatchCount());
-
- // test non-batch API
- c2 = RpcClientFactory.getInstance(p);
- Assert.assertTrue(c2 instanceof LoadBalancingRpcClient);
-
- for (int i = 0; i < 10; i++) {
- c2.append(getEvent(i));
- }
- Assert.assertEquals(10, h1.getAppendCount());
-
-
- } finally {
- if (s1 != null) s1.close();
- if (c1 != null) c1.close();
- if (c2 != null) c2.close();
}
- }
-
- @Test
- public void testTwoHostFailoverBatch() throws Exception {
- Server s1 = null;
- Server s2 = null;
- RpcClient c = null;
- try {
- LoadBalancedAvroHandler h1 = new LoadBalancedAvroHandler();
- LoadBalancedAvroHandler h2 = new LoadBalancedAvroHandler();
-
- s1 = RpcTestUtils.startServer(h1);
- s2 = RpcTestUtils.startServer(h2);
-
- Properties p = new Properties();
- p.put("hosts", "h1 h2");
- p.put("client.type", "default_loadbalance");
- p.put("hosts.h1", "127.0.0.1:" + s1.getPort());
- p.put("hosts.h2", "127.0.0.1:" + s2.getPort());
-
- c = RpcClientFactory.getInstance(p);
- Assert.assertTrue(c instanceof LoadBalancingRpcClient);
-
- for (int i = 0; i < 100; i++) {
- if (i == 20) {
- h2.setFailed();
- } else if (i == 40) {
- h2.setOK();
- }
-
- c.appendBatch(getBatchedEvent(i));
- }
- Assert.assertEquals(60, h1.getAppendBatchCount());
- Assert.assertEquals(40, h2.getAppendBatchCount());
- } finally {
- if (s1 != null) s1.close();
- if (s2 != null) s2.close();
- if (c != null) c.close();
- }
- }
-
- @Test
- public void testLbDefaultClientTwoHosts() throws Exception {
- Server s1 = null;
- Server s2 = null;
- RpcClient c = null;
- try {
- LoadBalancedAvroHandler h1 = new LoadBalancedAvroHandler();
- LoadBalancedAvroHandler h2 = new LoadBalancedAvroHandler();
-
- s1 = RpcTestUtils.startServer(h1);
- s2 = RpcTestUtils.startServer(h2);
-
- Properties p = new Properties();
- p.put("hosts", "h1 h2");
- p.put("client.type", "default_loadbalance");
- p.put("hosts.h1", "127.0.0.1:" + s1.getPort());
- p.put("hosts.h2", "127.0.0.1:" + s2.getPort());
-
- c = RpcClientFactory.getInstance(p);
- Assert.assertTrue(c instanceof LoadBalancingRpcClient);
-
- for (int i = 0; i < 100; i++) {
- c.append(getEvent(i));
- }
-
- Assert.assertEquals(50, h1.getAppendCount());
- Assert.assertEquals(50, h2.getAppendCount());
- } finally {
- if (s1 != null) s1.close();
- if (s2 != null) s2.close();
- if (c != null) c.close();
- }
- }
-
- @Test
- public void testLbDefaultClientTwoHostsBatch() throws Exception {
- Server s1 = null;
- Server s2 = null;
- RpcClient c = null;
- try {
- LoadBalancedAvroHandler h1 = new LoadBalancedAvroHandler();
- LoadBalancedAvroHandler h2 = new LoadBalancedAvroHandler();
-
- s1 = RpcTestUtils.startServer(h1);
- s2 = RpcTestUtils.startServer(h2);
-
- Properties p = new Properties();
- p.put("hosts", "h1 h2");
- p.put("client.type", "default_loadbalance");
- p.put("hosts.h1", "127.0.0.1:" + s1.getPort());
- p.put("hosts.h2", "127.0.0.1:" + s2.getPort());
-
- c = RpcClientFactory.getInstance(p);
- Assert.assertTrue(c instanceof LoadBalancingRpcClient);
-
- for (int i = 0; i < 100; i++) {
- c.appendBatch(getBatchedEvent(i));
- }
-
- Assert.assertEquals(50, h1.getAppendBatchCount());
- Assert.assertEquals(50, h2.getAppendBatchCount());
- } finally {
- if (s1 != null) s1.close();
- if (s2 != null) s2.close();
- if (c != null) c.close();
- }
- }
-
- @Test
- public void testLbClientTenHostRandomDistribution() throws Exception {
- final int NUM_HOSTS = 10;
- final int NUM_EVENTS = 1000;
- Server[] s = new Server[NUM_HOSTS];
- LoadBalancedAvroHandler[] h = new LoadBalancedAvroHandler[NUM_HOSTS];
- RpcClient c = null;
- try {
- Properties p = new Properties();
- StringBuilder hostList = new StringBuilder("");
- for (int i = 0; i < NUM_HOSTS; i++) {
- h[i] = new LoadBalancedAvroHandler();
- s[i] = RpcTestUtils.startServer(h[i]);
- String name = "h" + i;
- p.put("hosts." + name, "127.0.0.1:" + s[i].getPort());
- hostList.append(name).append(" ");
- }
-
- p.put("hosts", hostList.toString().trim());
- p.put("client.type", "default_loadbalance");
- p.put("host-selector", "random");
-
- c = RpcClientFactory.getInstance(p);
- Assert.assertTrue(c instanceof LoadBalancingRpcClient);
-
- for (int i = 0; i < NUM_EVENTS; i++) {
- c.append(getEvent(i));
- }
-
- Set<Integer> counts = new HashSet<Integer>();
- int total = 0;
- for (LoadBalancedAvroHandler handler : h) {
- total += handler.getAppendCount();
- counts.add(handler.getAppendCount());
- }
-
- Assert.assertTrue("Very unusual distribution", counts.size() > 2);
- Assert.assertTrue("Missing events", total == NUM_EVENTS);
- } finally {
- for (int i = 0; i < NUM_HOSTS; i++) {
- if (s[i] != null) s[i].close();
- }
+ // This will fail without FLUME-1823
+ @Test(expected = EventDeliveryException.class)
+ public void testTwoHostFailoverThrowAfterClose() throws Exception {
+ Server s1 = null;
+ Server s2 = null;
+ RpcClient c = null;
+ try {
+ LoadBalancedAvroHandler h1 = new LoadBalancedAvroHandler();
+ LoadBalancedAvroHandler h2 = new LoadBalancedAvroHandler();
+
+ s1 = RpcTestUtils.startServer(h1);
+ s2 = RpcTestUtils.startServer(h2);
+
+ Properties p = new Properties();
+ p.put("hosts", "h1 h2");
+ p.put("client.type", "default_loadbalance");
+ p.put("hosts.h1", "127.0.0.1:" + s1.getPort());
+ p.put("hosts.h2", "127.0.0.1:" + s2.getPort());
+
+ c = RpcClientFactory.getInstance(p);
+ Assert.assertTrue(c instanceof LoadBalancingRpcClient);
+
+ for (int i = 0; i < 100; i++) {
+ if (i == 20) {
+ h2.setFailed();
+ } else if (i == 40) {
+ h2.setOK();
+ }
+ c.append(getEvent(i));
+ }
+
+ Assert.assertEquals(60, h1.getAppendCount());
+ Assert.assertEquals(40, h2.getAppendCount());
+ if (c != null) c.close();
+ c.append(getEvent(3));
+ Assert.fail();
+ } finally {
+ if (s1 != null) s1.close();
+ if (s2 != null) s2.close();
+ }
}
- }
-
- @Test
- public void testLbClientTenHostRandomDistributionBatch() throws Exception {
- final int NUM_HOSTS = 10;
- final int NUM_EVENTS = 1000;
- Server[] s = new Server[NUM_HOSTS];
- LoadBalancedAvroHandler[] h = new LoadBalancedAvroHandler[NUM_HOSTS];
- RpcClient c = null;
- try {
- Properties p = new Properties();
- StringBuilder hostList = new StringBuilder("");
- for (int i = 0; i < NUM_HOSTS; i++) {
- h[i] = new LoadBalancedAvroHandler();
- s[i] = RpcTestUtils.startServer(h[i]);
- String name = "h" + i;
- p.put("hosts." + name, "127.0.0.1:" + s[i].getPort());
- hostList.append(name).append(" ");
- }
-
- p.put("hosts", hostList.toString().trim());
- p.put("client.type", "default_loadbalance");
- p.put("host-selector", "random");
-
- c = RpcClientFactory.getInstance(p);
... 20467 lines suppressed ...