You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@logging.apache.org by rg...@apache.org on 2024/05/13 00:44:01 UTC
(logging-flume) branch trunk updated: Remove the hadoop components
This is an automated email from the ASF dual-hosted git repository.
rgoers pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/logging-flume.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3e0984816 Remove the hadoop components
3e0984816 is described below
commit 3e098481668f0b47f0423be168448d3f7f5d8211
Author: Ralph Goers <rg...@apache.org>
AuthorDate: Sun May 12 16:48:28 2024 -0700
Remove the hadoop components
---
flume-bom/pom.xml | 18 +-
.../pom.xml | 59 -
.../HadoopCredentialStoreConfigFilter.java | 77 -
.../TestHadoopCredentialStoreConfigFilter.java | 183 ---
.../src/test/resources/test-password.txt | 1 -
.../src/test/resources/test-password2.txt | 1 -
flume-ng-configfilters/pom.xml | 1 -
flume-ng-dist/pom.xml | 10 +-
flume-ng-node/pom.xml | 5 +-
flume-ng-sinks/flume-hdfs-sink/pom.xml | 133 --
.../apache/flume/sink/hdfs/AbstractHDFSWriter.java | 281 ----
.../flume/sink/hdfs/AvroEventSerializer.java | 211 ---
.../flume/sink/hdfs/BucketClosedException.java | 30 -
.../org/apache/flume/sink/hdfs/BucketWriter.java | 776 ---------
.../flume/sink/hdfs/HDFSCompressedDataStream.java | 162 --
.../org/apache/flume/sink/hdfs/HDFSDataStream.java | 140 --
.../org/apache/flume/sink/hdfs/HDFSEventSink.java | 577 -------
.../apache/flume/sink/hdfs/HDFSSequenceFile.java | 122 --
.../apache/flume/sink/hdfs/HDFSTextSerializer.java | 79 -
.../flume/sink/hdfs/HDFSWritableSerializer.java | 77 -
.../org/apache/flume/sink/hdfs/HDFSWriter.java | 47 -
.../apache/flume/sink/hdfs/HDFSWriterFactory.java | 43 -
.../org/apache/flume/sink/hdfs/KerberosUser.java | 72 -
.../flume/sink/hdfs/SequenceFileSerializer.java | 68 -
.../sink/hdfs/SequenceFileSerializerFactory.java | 90 --
.../sink/hdfs/SequenceFileSerializerType.java | 37 -
.../apache/flume/sink/hdfs/HDFSBadDataStream.java | 45 -
.../apache/flume/sink/hdfs/HDFSTestSeqWriter.java | 77 -
.../flume/sink/hdfs/HDFSTestWriterFactory.java | 42 -
.../org/apache/flume/sink/hdfs/MockDataStream.java | 39 -
.../org/apache/flume/sink/hdfs/MockFileSystem.java | 139 --
.../flume/sink/hdfs/MockFsDataOutputStream.java | 49 -
.../org/apache/flume/sink/hdfs/MockHDFSWriter.java | 111 --
.../apache/flume/sink/hdfs/MyCustomSerializer.java | 58 -
.../flume/sink/hdfs/TestAvroEventSerializer.java | 183 ---
.../apache/flume/sink/hdfs/TestBucketWriter.java | 719 ---------
.../sink/hdfs/TestHDFSCompressedDataStream.java | 141 --
.../apache/flume/sink/hdfs/TestHDFSEventSink.java | 1687 --------------------
.../flume/sink/hdfs/TestHDFSEventSinkDeadlock.java | 127 --
.../sink/hdfs/TestHDFSEventSinkOnMiniCluster.java | 607 -------
.../hdfs/TestSequenceFileSerializerFactory.java | 58 -
.../flume/sink/hdfs/TestUseRawLocalFileSystem.java | 95 --
.../flume-hdfs-sink/src/test/resources/log4j2.xml | 33 -
flume-ng-sinks/flume-hive-sink/pom.xml | 149 --
.../java/org/apache/flume/sink/hive/Config.java | 41 -
.../sink/hive/HiveDelimitedTextSerializer.java | 115 --
.../flume/sink/hive/HiveEventSerializer.java | 41 -
.../apache/flume/sink/hive/HiveJsonSerializer.java | 62 -
.../java/org/apache/flume/sink/hive/HiveSink.java | 528 ------
.../org/apache/flume/sink/hive/HiveWriter.java | 513 ------
.../org/apache/flume/sink/hive/TestHiveSink.java | 413 -----
.../org/apache/flume/sink/hive/TestHiveWriter.java | 350 ----
.../java/org/apache/flume/sink/hive/TestUtil.java | 235 ---
.../flume-hive-sink/src/test/resources/log4j2.xml | 33 -
flume-ng-sinks/flume-ng-hbase2-sink/pom.xml | 199 ---
.../org/apache/flume/sink/hbase2/BatchAware.java | 28 -
.../flume/sink/hbase2/HBase2EventSerializer.java | 61 -
.../org/apache/flume/sink/hbase2/HBase2Sink.java | 555 -------
.../hbase2/HBase2SinkConfigurationConstants.java | 77 -
.../sink/hbase2/RegexHBase2EventSerializer.java | 214 ---
.../sink/hbase2/SimpleHBase2EventSerializer.java | 151 --
.../flume/sink/hbase2/SimpleRowKeyGenerator.java | 46 -
.../sink/hbase2/IncrementHBase2Serializer.java | 83 -
.../hbase2/MockSimpleHBase2EventSerializer.java | 38 -
.../apache/flume/sink/hbase2/TestHBase2Sink.java | 786 ---------
.../flume/sink/hbase2/TestHBase2SinkCreation.java | 48 -
.../hbase2/TestRegexHBase2EventSerializer.java | 238 ---
flume-ng-sinks/flume-ng-kudu-sink/pom.xml | 195 ---
.../sink/kudu/AvroKuduOperationsProducer.java | 282 ----
.../flume/sink/kudu/KuduOperationsProducer.java | 54 -
.../java/org/apache/flume/sink/kudu/KuduSink.java | 315 ----
.../sink/kudu/KuduSinkConfigurationConstants.java | 77 -
.../sink/kudu/RegexpKuduOperationsProducer.java | 398 -----
.../kudu/SimpleKeyedKuduOperationsProducer.java | 136 --
.../sink/kudu/SimpleKuduOperationsProducer.java | 92 --
.../apache/flume/sink/kudu/KuduSinkTestUtil.java | 99 --
.../sink/kudu/TestAvroKuduOperationsProducer.java | 183 ---
.../sink/kudu/TestKeyedKuduOperationsProducer.java | 189 ---
.../org/apache/flume/sink/kudu/TestKuduSink.java | 213 ---
.../kudu/TestRegexpKuduOperationsProducer.java | 184 ---
...TestRegexpKuduOperationsProducerParseError.java | 291 ----
.../apache/flume/sink/kudu/TestSecureKuduSink.java | 126 --
.../src/test/resources/log4j2.xml | 32 -
.../resources/testAvroKuduOperationsProducer.avsc | 11 -
flume-ng-sinks/pom.xml | 24 -
85 files changed, 17 insertions(+), 15398 deletions(-)
diff --git a/flume-bom/pom.xml b/flume-bom/pom.xml
index ee4e7bf23..2d803c846 100644
--- a/flume-bom/pom.xml
+++ b/flume-bom/pom.xml
@@ -32,7 +32,7 @@ limitations under the License.
<name>Apache Flume Artifacts</name>
<properties>
- <flume-hadoop.version>1.11.1-SNAPSHOT</flume-hadoop.version>
+ <flume-hadoop.version>2.0.0-SNAPSHOT</flume-hadoop.version>
<flume-http.version>1.11.1-SNAPSHOT</flume-http.version>
<flume-irc.version>1.11.1-SNAPSHOT</flume-irc.version>
<flume-jdbc.version>2.0.0-SNAPSHOT</flume-jdbc.version>
@@ -92,8 +92,8 @@ limitations under the License.
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-hadoop-credential-store-config-filter</artifactId>
- <version>${project.version}</version>
+ <artifactId>flume-hadoop-credential-store-config-filter</artifactId>
+ <version>${flume-hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
@@ -172,12 +172,12 @@ limitations under the License.
<version>${morphline.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.flume.flume-ng-sinks</groupId>
+ <groupId>org.apache.flume</groupId>
<artifactId>flume-hdfs-sink</artifactId>
<version>${flume-hadoop.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.flume.flume-ng-sinks</groupId>
+ <groupId>org.apache.flume</groupId>
<artifactId>flume-hive-sink</artifactId>
<version>${flume-hadoop.version}</version>
</dependency>
@@ -192,8 +192,8 @@ limitations under the License.
<version>${flume-irc.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.flume.flume-ng-sinks</groupId>
- <artifactId>flume-ng-hbase2-sink</artifactId>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-hbase2-sink</artifactId>
<version>${flume-hadoop.version}</version>
</dependency>
<dependency>
@@ -202,8 +202,8 @@ limitations under the License.
<version>${flume-kafka.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.flume.flume-ng-sinks</groupId>
- <artifactId>flume-ng-kudu-sink</artifactId>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-kudu-sink</artifactId>
<version>${flume-hadoop.version}</version>
</dependency>
<dependency>
diff --git a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/pom.xml b/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/pom.xml
deleted file mode 100644
index a744bb20f..000000000
--- a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/pom.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>flume-ng-configfilters</artifactId>
- <groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <properties>
- <module.name>org.apache.flume.configfilter.credstore</module.name>
- </properties>
-
- <artifactId>flume-ng-hadoop-credential-store-config-filter</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-config-filter-api</artifactId>
- </dependency>
- <!-- test dependencies -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>com.github.stefanbirkner</groupId>
- <artifactId>system-rules</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
-
-</project>
\ No newline at end of file
diff --git a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/main/java/org/apache/flume/configfilter/HadoopCredentialStoreConfigFilter.java b/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/main/java/org/apache/flume/configfilter/HadoopCredentialStoreConfigFilter.java
deleted file mode 100644
index c8d68703c..000000000
--- a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/main/java/org/apache/flume/configfilter/HadoopCredentialStoreConfigFilter.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.flume.configfilter;
-
-import java.io.IOException;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HadoopCredentialStoreConfigFilter extends AbstractConfigFilter {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(
- HadoopCredentialStoreConfigFilter.class);
-
- static final String PASSWORD_FILE_CONFIG_KEY
- = "credstore.java-keystore-provider.password-file";
- static final String CREDENTIAL_PROVIDER_PATH
- = "credential.provider.path";
- static final String HADOOP_SECURITY = "hadoop.security.";
- private Configuration hadoopConfiguration;
-
- public void initializeWithConfiguration(Map<String, String> configuration) {
-
- LOGGER.debug("Initializing hadoop credential store.");
- hadoopConfiguration = new Configuration();
- hadoopConfiguration.set(
- HADOOP_SECURITY + CREDENTIAL_PROVIDER_PATH,
- configuration.get(CREDENTIAL_PROVIDER_PATH)
- );
-
- String passwordFile = configuration.get(PASSWORD_FILE_CONFIG_KEY);
- if (passwordFile != null && !passwordFile.isEmpty()) {
- checkPasswordFile(passwordFile);
- hadoopConfiguration.set(
- HADOOP_SECURITY + PASSWORD_FILE_CONFIG_KEY, passwordFile
- );
- }
- }
-
- private void checkPasswordFile(String passwordFile) {
- if (Thread.currentThread().getContextClassLoader().getResource(passwordFile) == null) {
- LOGGER.error("The java keystore provider password file has to be on the classpath." +
- " The password file provided in the configuration cannot be found and will not be used"
- );
- }
- }
-
- @Override
- public String filter(String key) {
- char[] result = null;
- try {
- result = hadoopConfiguration.getPassword(key);
- } catch (IOException e) {
- LOGGER.error("Error while reading value for key {}: ", key, e);
- }
-
- return result == null ? null : new String(result);
- }
-
-
-}
diff --git a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/java/org/apache/flume/configfilter/TestHadoopCredentialStoreConfigFilter.java b/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/java/org/apache/flume/configfilter/TestHadoopCredentialStoreConfigFilter.java
deleted file mode 100644
index 55f74d8cf..000000000
--- a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/java/org/apache/flume/configfilter/TestHadoopCredentialStoreConfigFilter.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.flume.configfilter;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.alias.CredentialShell;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.contrib.java.lang.system.EnvironmentVariables;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.util.HashMap;
-
-import static org.apache.flume.configfilter.HadoopCredentialStoreConfigFilter.CREDENTIAL_PROVIDER_PATH;
-import static org.apache.flume.configfilter.HadoopCredentialStoreConfigFilter.HADOOP_SECURITY;
-import static org.apache.flume.configfilter.HadoopCredentialStoreConfigFilter.PASSWORD_FILE_CONFIG_KEY;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-
-public class TestHadoopCredentialStoreConfigFilter {
-
- private static String providerPathDefault;
- private static String providerPathEnv;
- private static String providerPathPwdFile;
-
- @ClassRule
- public static final EnvironmentVariables environmentVariables
- = new EnvironmentVariables();
- private static File fileDefault;
- private static File fileEnvPassword;
- private static File fileFilePassword;
- private HadoopCredentialStoreConfigFilter configFilter;
-
- @BeforeClass
- public static void setUpClass() throws Exception {
- generateTempFileNames();
- fillCredStoreWithDefaultPassword();
- fillCredStoreWithPasswordFile();
- fillCredStoreWithEnvironmentVariablePassword();
- }
- @AfterClass
- public static void tearDown() {
- fileDefault.deleteOnExit();
- fileEnvPassword.deleteOnExit();
- fileFilePassword.deleteOnExit();
- }
-
- @Before
- public void setUp() {
- String[] objects = System.getenv().keySet().toArray(new String[0]);
- environmentVariables.clear(objects);
- configFilter = new HadoopCredentialStoreConfigFilter();
- }
-
- @Test
- public void filterDefaultPasswordFile() {
- HashMap<String, String> configuration = new HashMap<>();
- configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathDefault);
- configFilter.initializeWithConfiguration(configuration);
-
- assertEquals("filtered_default", configFilter.filter("password"));
- }
-
- @Test
- public void filterWithEnvPassword() {
- environmentVariables.set("HADOOP_CREDSTORE_PASSWORD","envSecret");
- HashMap<String, String> configuration = new HashMap<>();
- configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathEnv);
- configFilter.initializeWithConfiguration(configuration);
-
- assertEquals("filtered_env", configFilter.filter("password"));
- }
-
- @Test
- public void filterWithPasswordFile() {
- HashMap<String, String> configuration = new HashMap<>();
- configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathPwdFile);
- configuration.put(PASSWORD_FILE_CONFIG_KEY, "test-password.txt");
- configFilter.initializeWithConfiguration(configuration);
-
- assertEquals("filtered_file", configFilter.filter("password"));
- }
-
- @Test
- public void filterWithEnvNoPassword() {
- HashMap<String, String> configuration = new HashMap<>();
- configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathEnv);
- configFilter.initializeWithConfiguration(configuration);
-
- assertNull(configFilter.filter("password"));
- }
-
- @Test
- public void filterErrorWithPasswordFileWrongPassword() {
- HashMap<String, String> configuration = new HashMap<>();
- configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathPwdFile);
- configuration.put(PASSWORD_FILE_CONFIG_KEY, "test-password2.txt");
- configFilter.initializeWithConfiguration(configuration);
-
- assertNull(configFilter.filter("password"));
- }
-
- @Test
- public void filterErrorWithPasswordFileNoPasswordFile() {
- HashMap<String, String> configuration = new HashMap<>();
- configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathPwdFile);
- configFilter.initializeWithConfiguration(configuration);
-
- assertNull(configFilter.filter("password"));
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void filterErrorWithNoProvider() {
- HashMap<String, String> configuration = new HashMap<>();
- configFilter.initializeWithConfiguration(configuration);
- }
-
-
- private static void fillCredStoreWithEnvironmentVariablePassword() throws Exception {
- environmentVariables.set("HADOOP_CREDSTORE_PASSWORD", "envSecret");
-
- runCommand("create password -value filtered_env -provider "
- + providerPathEnv, new Configuration());
- }
-
- private static void fillCredStoreWithPasswordFile() throws Exception {
- Configuration conf = new Configuration();
- conf.set(
- HADOOP_SECURITY + PASSWORD_FILE_CONFIG_KEY,
- "test-password.txt"
- );
- runCommand("create password -value filtered_file -provider "
- + providerPathPwdFile, conf);
- }
-
- private static void fillCredStoreWithDefaultPassword() throws Exception {
- runCommand("create password -value filtered_default -provider "
- + providerPathDefault, new Configuration());
- }
-
- private static void generateTempFileNames() throws IOException {
- fileDefault = Files.createTempFile("test-default-pwd-", ".jceks").toFile();
- boolean deleted = fileDefault.delete();
- fileEnvPassword = Files.createTempFile("test-env-pwd-", ".jceks").toFile();
- deleted &= fileEnvPassword.delete();
- fileFilePassword = Files.createTempFile("test-file-pwd-", ".jceks").toFile();
- deleted &= fileFilePassword.delete();
- if (!deleted) {
- fail("Could not delete temporary files");
- }
-
- providerPathDefault = "jceks://file/" + fileDefault.getAbsolutePath();
- providerPathEnv = "jceks://file/" + fileEnvPassword.getAbsolutePath();
- providerPathPwdFile = "jceks://file/" + fileFilePassword.getAbsolutePath();
- }
-
- private static void runCommand(String c, Configuration conf) throws Exception {
- ToolRunner.run(conf, new CredentialShell(), c.split(" "));
- }
-
-
-}
\ No newline at end of file
diff --git a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/resources/test-password.txt b/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/resources/test-password.txt
deleted file mode 100644
index db5fd9185..000000000
--- a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/resources/test-password.txt
+++ /dev/null
@@ -1 +0,0 @@
-fileSecret
\ No newline at end of file
diff --git a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/resources/test-password2.txt b/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/resources/test-password2.txt
deleted file mode 100644
index 2dee0b131..000000000
--- a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/resources/test-password2.txt
+++ /dev/null
@@ -1 +0,0 @@
-fileSecret2
\ No newline at end of file
diff --git a/flume-ng-configfilters/pom.xml b/flume-ng-configfilters/pom.xml
index 1ed177500..7cc35884a 100644
--- a/flume-ng-configfilters/pom.xml
+++ b/flume-ng-configfilters/pom.xml
@@ -31,7 +31,6 @@ limitations under the License.
<modules>
<module>flume-ng-environment-variable-config-filter</module>
- <module>flume-ng-hadoop-credential-store-config-filter</module>
<module>flume-ng-config-filter-api</module>
<module>flume-ng-external-process-config-filter</module>
</modules>
diff --git a/flume-ng-dist/pom.xml b/flume-ng-dist/pom.xml
index 7be10b2f1..373b499fe 100644
--- a/flume-ng-dist/pom.xml
+++ b/flume-ng-dist/pom.xml
@@ -133,7 +133,7 @@
<artifactId>flume-kafka-channel</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.flume.flume-ng-sinks</groupId>
+ <groupId>org.apache.flume</groupId>
<artifactId>flume-hdfs-sink</artifactId>
</dependency>
<dependency>
@@ -141,8 +141,8 @@
<artifactId>flume-irc-sink</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.flume.flume-ng-sinks</groupId>
- <artifactId>flume-ng-hbase2-sink</artifactId>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-hbase2-sink</artifactId>
</dependency>
<dependency>
<groupId>org.apache.flume.flume-ng-sinks</groupId>
@@ -157,7 +157,7 @@
<artifactId>flume-kafka-sink</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.flume.flume-ng-sinks</groupId>
+ <groupId>org.apache.flume</groupId>
<artifactId>flume-hive-sink</artifactId>
</dependency>
<dependency>
@@ -194,7 +194,7 @@
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-hadoop-credential-store-config-filter</artifactId>
+ <artifactId>flume-hadoop-credential-store-config-filter</artifactId>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
diff --git a/flume-ng-node/pom.xml b/flume-ng-node/pom.xml
index 00d77e2f6..2f7453f66 100644
--- a/flume-ng-node/pom.xml
+++ b/flume-ng-node/pom.xml
@@ -56,10 +56,11 @@
<artifactId>flume-ng-core</artifactId>
</dependency>
+ <!--
<dependency>
- <groupId>org.apache.flume.flume-ng-sinks</groupId>
+ <groupId>org.apache.flume</groupId>
<artifactId>flume-hdfs-sink</artifactId>
- </dependency>
+ </dependency>-->
<dependency>
<groupId>org.apache.flume.flume-ng-sinks</groupId>
diff --git a/flume-ng-sinks/flume-hdfs-sink/pom.xml b/flume-ng-sinks/flume-hdfs-sink/pom.xml
deleted file mode 100644
index 07404c1cc..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/pom.xml
+++ /dev/null
@@ -1,133 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>flume-ng-sinks</artifactId>
- <groupId>org.apache.flume</groupId>
- <version>1.11.1-SNAPSHOT</version>
- </parent>
-
- <groupId>org.apache.flume.flume-ng-sinks</groupId>
- <artifactId>flume-hdfs-sink</artifactId>
- <name>Flume NG HDFS Sink</name>
-
- <properties>
- <!-- TODO fix spotbugs/pmd violations -->
- <spotbugs.maxAllowedViolations>14</spotbugs.maxAllowedViolations>
- <pmd.maxAllowedViolations>25</pmd.maxAllowedViolations>
- <module.name>org.apache.flume.sink.hdfs</module.name>
- </properties>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-sdk</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-configuration</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-client</artifactId>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
-
- </dependencies>
-
-</project>
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AbstractHDFSWriter.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AbstractHDFSWriter.java
deleted file mode 100644
index d2a07faf1..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AbstractHDFSWriter.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import com.google.common.base.Preconditions;
-import org.apache.flume.Context;
-import org.apache.flume.FlumeException;
-import org.apache.flume.annotations.InterfaceAudience;
-import org.apache.flume.annotations.InterfaceStability;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public abstract class AbstractHDFSWriter implements HDFSWriter {
-
- private static final Logger logger =
- LoggerFactory.getLogger(AbstractHDFSWriter.class);
-
- private FSDataOutputStream outputStream;
- private FileSystem fs;
- private Path destPath;
- private Method refGetNumCurrentReplicas = null;
- private Method refGetDefaultReplication = null;
- private Method refHflushOrSync = null;
- private Integer configuredMinReplicas = null;
- private Integer numberOfCloseRetries = null;
- private long timeBetweenCloseRetries = Long.MAX_VALUE;
-
- static final Object[] NO_ARGS = new Object[]{};
-
- @Override
- public void configure(Context context) {
- configuredMinReplicas = context.getInteger("hdfs.minBlockReplicas");
- if (configuredMinReplicas != null) {
- Preconditions.checkArgument(configuredMinReplicas >= 0,
- "hdfs.minBlockReplicas must be greater than or equal to 0");
- }
- numberOfCloseRetries = context.getInteger("hdfs.closeTries", 1) - 1;
-
- if (numberOfCloseRetries > 1) {
- try {
- //hdfs.callTimeout is deprecated from 1.9
- timeBetweenCloseRetries = context.getLong("hdfs.callTimeout", 30000L);
- } catch (NumberFormatException e) {
- logger.warn("hdfs.callTimeout can not be parsed to a long: " +
- context.getLong("hdfs.callTimeout"));
- }
- timeBetweenCloseRetries = Math.max(timeBetweenCloseRetries / numberOfCloseRetries, 1000);
- }
-
- }
-
- /**
- * Contract for subclasses: Call registerCurrentStream() on open,
- * unregisterCurrentStream() on close, and the base class takes care of the
- * rest.
- * @return
- */
- @Override
- public boolean isUnderReplicated() {
- try {
- int numBlocks = getNumCurrentReplicas();
- if (numBlocks == -1) {
- return false;
- }
- int desiredBlocks;
- if (configuredMinReplicas != null) {
- desiredBlocks = configuredMinReplicas;
- } else {
- desiredBlocks = getFsDesiredReplication();
- }
- return numBlocks < desiredBlocks;
- } catch (IllegalAccessException e) {
- logger.error("Unexpected error while checking replication factor", e);
- } catch (InvocationTargetException e) {
- logger.error("Unexpected error while checking replication factor", e);
- } catch (IllegalArgumentException e) {
- logger.error("Unexpected error while checking replication factor", e);
- }
- return false;
- }
-
- protected void registerCurrentStream(FSDataOutputStream outputStream,
- FileSystem fs, Path destPath) {
- Preconditions.checkNotNull(outputStream, "outputStream must not be null");
- Preconditions.checkNotNull(fs, "fs must not be null");
- Preconditions.checkNotNull(destPath, "destPath must not be null");
-
- this.outputStream = outputStream;
- this.fs = fs;
- this.destPath = destPath;
- this.refGetNumCurrentReplicas = reflectGetNumCurrentReplicas(outputStream);
- this.refGetDefaultReplication = reflectGetDefaultReplication(fs);
- this.refHflushOrSync = reflectHflushOrSync(outputStream);
-
- }
-
- protected void unregisterCurrentStream() {
- this.outputStream = null;
- this.fs = null;
- this.destPath = null;
- this.refGetNumCurrentReplicas = null;
- this.refGetDefaultReplication = null;
- }
-
- public int getFsDesiredReplication() {
- short replication = 0;
- if (fs != null && destPath != null) {
- if (refGetDefaultReplication != null) {
- try {
- replication = (Short) refGetDefaultReplication.invoke(fs, destPath);
- } catch (IllegalAccessException e) {
- logger.warn("Unexpected error calling getDefaultReplication(Path)", e);
- } catch (InvocationTargetException e) {
- logger.warn("Unexpected error calling getDefaultReplication(Path)", e);
- }
- } else {
- // will not work on Federated HDFS (see HADOOP-8014)
- replication = fs.getDefaultReplication();
- }
- }
- return replication;
- }
-
- /**
- * This method gets the datanode replication count for the current open file.
- *
- * If the pipeline isn't started yet or is empty, you will get the default
- * replication factor.
- *
- * <p/>If this function returns -1, it means you
- * are not properly running with the HDFS-826 patch.
- * @throws InvocationTargetException
- * @throws IllegalAccessException
- * @throws IllegalArgumentException
- */
- public int getNumCurrentReplicas()
- throws IllegalArgumentException, IllegalAccessException,
- InvocationTargetException {
- if (refGetNumCurrentReplicas != null && outputStream != null) {
- OutputStream dfsOutputStream = outputStream.getWrappedStream();
- if (dfsOutputStream != null) {
- Object repl = refGetNumCurrentReplicas.invoke(dfsOutputStream, NO_ARGS);
- if (repl instanceof Integer) {
- return ((Integer)repl).intValue();
- }
- }
- }
- return -1;
- }
-
- /**
- * Find the 'getNumCurrentReplicas' on the passed <code>os</code> stream.
- * @return Method or null.
- */
- private Method reflectGetNumCurrentReplicas(FSDataOutputStream os) {
- Method m = null;
- if (os != null) {
- Class<? extends OutputStream> wrappedStreamClass = os.getWrappedStream()
- .getClass();
- try {
- m = wrappedStreamClass.getDeclaredMethod("getNumCurrentReplicas",
- new Class<?>[] {});
- m.setAccessible(true);
- } catch (NoSuchMethodException e) {
- logger.info("FileSystem's output stream doesn't support"
- + " getNumCurrentReplicas; --HDFS-826 not available; fsOut="
- + wrappedStreamClass.getName() + "; err=" + e);
- } catch (SecurityException e) {
- logger.info("Doesn't have access to getNumCurrentReplicas on "
- + "FileSystems's output stream --HDFS-826 not available; fsOut="
- + wrappedStreamClass.getName(), e);
- m = null; // could happen on setAccessible()
- }
- }
- if (m != null) {
- logger.debug("Using getNumCurrentReplicas--HDFS-826");
- }
- return m;
- }
-
- /**
- * Find the 'getDefaultReplication' method on the passed <code>fs</code>
- * FileSystem that takes a Path argument.
- * @return Method or null.
- */
- private Method reflectGetDefaultReplication(FileSystem fileSystem) {
- Method m = null;
- if (fileSystem != null) {
- Class<?> fsClass = fileSystem.getClass();
- try {
- m = fsClass.getMethod("getDefaultReplication",
- new Class<?>[] { Path.class });
- } catch (NoSuchMethodException e) {
- logger.debug("FileSystem implementation doesn't support"
- + " getDefaultReplication(Path); -- HADOOP-8014 not available; " +
- "className = " + fsClass.getName() + "; err = " + e);
- } catch (SecurityException e) {
- logger.debug("No access to getDefaultReplication(Path) on "
- + "FileSystem implementation -- HADOOP-8014 not available; " +
- "className = " + fsClass.getName() + "; err = " + e);
- }
- }
- if (m != null) {
- logger.debug("Using FileSystem.getDefaultReplication(Path) from " +
- "HADOOP-8014");
- }
- return m;
- }
-
- private Method reflectHflushOrSync(FSDataOutputStream os) {
- Method m = null;
- if (os != null) {
- Class<?> fsDataOutputStreamClass = os.getClass();
- try {
- m = fsDataOutputStreamClass.getMethod("hflush");
- } catch (NoSuchMethodException ex) {
- logger.debug("HFlush not found. Will use sync() instead");
- try {
- m = fsDataOutputStreamClass.getMethod("sync");
- } catch (Exception ex1) {
- String msg = "Neither hflush not sync were found. That seems to be " +
- "a problem!";
- logger.error(msg);
- throw new FlumeException(msg, ex1);
- }
- }
- }
- return m;
- }
-
- /**
- * If hflush is available in this version of HDFS, then this method calls
- * hflush, else it calls sync.
- * @param os - The stream to flush/sync
- * @throws IOException
- */
- protected void hflushOrSync(FSDataOutputStream os) throws IOException {
- try {
- // At this point the refHflushOrSync cannot be null,
- // since register method would have thrown if it was.
- this.refHflushOrSync.invoke(os);
- } catch (InvocationTargetException e) {
- String msg = "Error while trying to hflushOrSync!";
- logger.error(msg);
- Throwable cause = e.getCause();
- if (cause != null && cause instanceof IOException) {
- throw (IOException)cause;
- }
- throw new FlumeException(msg, e);
- } catch (Exception e) {
- String msg = "Error while trying to hflushOrSync!";
- logger.error(msg);
- throw new FlumeException(msg, e);
- }
- }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AvroEventSerializer.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AvroEventSerializer.java
deleted file mode 100644
index 323174240..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AvroEventSerializer.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import org.apache.avro.AvroRuntimeException;
-import org.apache.avro.Schema;
-import org.apache.avro.file.CodecFactory;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.avro.io.DatumWriter;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.FlumeException;
-import org.apache.flume.conf.Configurable;
-import org.apache.flume.serialization.EventSerializer;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URL;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-
-import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.COMPRESSION_CODEC;
-import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_COMPRESSION_CODEC;
-import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_STATIC_SCHEMA_URL;
-import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_SYNC_INTERVAL_BYTES;
-import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.STATIC_SCHEMA_URL;
-import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.SYNC_INTERVAL_BYTES;
-
-/**
- * <p>
- * This class serializes Flume {@linkplain org.apache.flume.Event events} into Avro data files. The
- * Flume event body is read as an Avro datum, and is then written to the
- * {@link org.apache.flume.serialization.EventSerializer}'s output stream in Avro data file format.
- * </p>
- * <p>
- * The Avro schema is determined by reading a Flume event header. The schema may be
- * specified either as a literal, by setting {@link #AVRO_SCHEMA_LITERAL_HEADER} (not
- * recommended, since the full schema must be transmitted in every event),
- * or as a URL which the schema may be read from, by setting {@link
- * #AVRO_SCHEMA_URL_HEADER}. Schemas read from URLs are cached by instances of this
- * class so that the overhead of retrieval is minimized.
- * </p>
- */
-public class AvroEventSerializer implements EventSerializer, Configurable {
-
- private static final Logger logger =
- LoggerFactory.getLogger(AvroEventSerializer.class);
-
- public static final String AVRO_SCHEMA_LITERAL_HEADER = "flume.avro.schema.literal";
- public static final String AVRO_SCHEMA_URL_HEADER = "flume.avro.schema.url";
-
- private final OutputStream out;
- private DatumWriter<Object> writer = null;
- private DataFileWriter<Object> dataFileWriter = null;
-
- private int syncIntervalBytes;
- private String compressionCodec;
- private Map<String, Schema> schemaCache = new HashMap<String, Schema>();
- private String staticSchemaURL;
-
- private AvroEventSerializer(OutputStream out) {
- this.out = out;
- }
-
- @Override
- public void configure(Context context) {
- syncIntervalBytes =
- context.getInteger(SYNC_INTERVAL_BYTES, DEFAULT_SYNC_INTERVAL_BYTES);
- compressionCodec =
- context.getString(COMPRESSION_CODEC, DEFAULT_COMPRESSION_CODEC);
- staticSchemaURL = context.getString(STATIC_SCHEMA_URL, DEFAULT_STATIC_SCHEMA_URL);
- }
-
- @Override
- public void afterCreate() throws IOException {
- // no-op
- }
-
- @Override
- public void afterReopen() throws IOException {
- // impossible to initialize DataFileWriter without writing the schema?
- throw new UnsupportedOperationException("Avro API doesn't support append");
- }
-
- @Override
- public void write(Event event) throws IOException {
- if (dataFileWriter == null) {
- initialize(event);
- }
- dataFileWriter.appendEncoded(ByteBuffer.wrap(event.getBody()));
- }
-
- private void initialize(Event event) throws IOException {
- Schema schema = null;
- String schemaUrl = event.getHeaders().get(AVRO_SCHEMA_URL_HEADER);
- String schemaString = event.getHeaders().get(AVRO_SCHEMA_LITERAL_HEADER);
-
- if (schemaUrl != null) { // if URL_HEADER is there then use it
- schema = schemaCache.get(schemaUrl);
- if (schema == null) {
- schema = loadFromUrl(schemaUrl);
- schemaCache.put(schemaUrl, schema);
- }
- } else if (schemaString != null) { // fallback to LITERAL_HEADER if it was there
- schema = new Schema.Parser().parse(schemaString);
- } else if (staticSchemaURL != null) { // fallback to static url if it was there
- schema = schemaCache.get(staticSchemaURL);
- if (schema == null) {
- schema = loadFromUrl(staticSchemaURL);
- schemaCache.put(staticSchemaURL, schema);
- }
- } else { // no other options so giving up
- throw new FlumeException("Could not find schema for event " + event);
- }
-
- writer = new GenericDatumWriter<Object>(schema);
- dataFileWriter = new DataFileWriter<Object>(writer);
-
- dataFileWriter.setSyncInterval(syncIntervalBytes);
-
- try {
- CodecFactory codecFactory = CodecFactory.fromString(compressionCodec);
- dataFileWriter.setCodec(codecFactory);
- } catch (AvroRuntimeException e) {
- logger.warn("Unable to instantiate avro codec with name (" +
- compressionCodec + "). Compression disabled. Exception follows.", e);
- }
-
- dataFileWriter.create(schema, out);
- }
-
- private Schema loadFromUrl(String schemaUrl) throws IOException {
- Configuration conf = new Configuration();
- Schema.Parser parser = new Schema.Parser();
- if (schemaUrl.toLowerCase(Locale.ENGLISH).startsWith("hdfs://")) {
- FileSystem fs = FileSystem.get(conf);
- FSDataInputStream input = null;
- try {
- input = fs.open(new Path(schemaUrl));
- return parser.parse(input);
- } finally {
- if (input != null) {
- input.close();
- }
- }
- } else {
- InputStream is = null;
- try {
- is = new URL(schemaUrl).openStream();
- return parser.parse(is);
- } finally {
- if (is != null) {
- is.close();
- }
- }
- }
- }
-
- @Override
- public void flush() throws IOException {
- dataFileWriter.flush();
- }
-
- @Override
- public void beforeClose() throws IOException {
- // no-op
- }
-
- @Override
- public boolean supportsReopen() {
- return false;
- }
-
- public static class Builder implements EventSerializer.Builder {
-
- @Override
- public EventSerializer build(Context context, OutputStream out) {
- AvroEventSerializer writer = new AvroEventSerializer(out);
- writer.configure(context);
- return writer;
- }
-
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketClosedException.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketClosedException.java
deleted file mode 100644
index 1d8a9e4c5..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketClosedException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import org.apache.flume.FlumeException;
-
-public class BucketClosedException extends FlumeException {
-
- private static final long serialVersionUID = -4216667125119540357L;
-
- public BucketClosedException(String msg) {
- super(msg);
- }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketWriter.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketWriter.java
deleted file mode 100644
index c3941a692..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketWriter.java
+++ /dev/null
@@ -1,776 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Throwables;
-import org.apache.flume.Clock;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.SystemClock;
-import org.apache.flume.auth.PrivilegedExecutor;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.flume.sink.hdfs.HDFSEventSink.WriterCallback;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.security.PrivilegedExceptionAction;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Internal API intended for HDFSSink use.
- * This class does file rolling and handles file formats and serialization.
- * Only the public methods in this class are thread safe.
- */
-class BucketWriter {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(BucketWriter.class);
-
- /**
- * This lock ensures that only one thread can open a file at a time.
- */
- private static final Integer staticLock = new Integer(1);
- private Method isClosedMethod = null;
-
- private final HDFSWriter writer;
- private final long rollInterval;
- private final long rollSize;
- private final long rollCount;
- private final long batchSize;
- private final CompressionCodec codeC;
- private final CompressionType compType;
- private final ScheduledExecutorService timedRollerPool;
- private final PrivilegedExecutor proxyUser;
-
- private final AtomicLong fileExtensionCounter;
-
- private long eventCounter;
- private long processSize;
-
- private FileSystem fileSystem;
-
- private volatile String filePath;
- private volatile String fileName;
- private volatile String inUsePrefix;
- private volatile String inUseSuffix;
- private volatile String fileSuffix;
- private volatile String bucketPath;
- private volatile String targetPath;
- private volatile long batchCounter;
- private volatile boolean isOpen;
- private volatile boolean isUnderReplicated;
- private volatile int consecutiveUnderReplRotateCount = 0;
- private volatile ScheduledFuture<Void> timedRollFuture;
- private SinkCounter sinkCounter;
- private final int idleTimeout;
- private volatile ScheduledFuture<Void> idleFuture;
- private final WriterCallback onCloseCallback;
- private final String onCloseCallbackPath;
- private final long callTimeout;
- private final ExecutorService callTimeoutPool;
- private final int maxConsecUnderReplRotations = 30; // make this config'able?
-
- private boolean mockFsInjected = false;
-
- private final long retryInterval;
- private final int maxRetries;
-
- // flag that the bucket writer was closed due to idling and thus shouldn't be
- // reopened. Not ideal, but avoids internals of owners
- protected AtomicBoolean closed = new AtomicBoolean();
- AtomicInteger renameTries = new AtomicInteger(0);
-
- BucketWriter(long rollInterval, long rollSize, long rollCount, long batchSize,
- Context context, String filePath, String fileName, String inUsePrefix,
- String inUseSuffix, String fileSuffix, CompressionCodec codeC,
- CompressionType compType, HDFSWriter writer,
- ScheduledExecutorService timedRollerPool, PrivilegedExecutor proxyUser,
- SinkCounter sinkCounter, int idleTimeout, WriterCallback onCloseCallback,
- String onCloseCallbackPath, long callTimeout,
- ExecutorService callTimeoutPool, long retryInterval,
- int maxRetries) {
- this(rollInterval, rollSize, rollCount, batchSize,
- context, filePath, fileName, inUsePrefix,
- inUseSuffix, fileSuffix, codeC,
- compType, writer,
- timedRollerPool, proxyUser,
- sinkCounter, idleTimeout, onCloseCallback,
- onCloseCallbackPath, callTimeout,
- callTimeoutPool, retryInterval,
- maxRetries, new SystemClock());
- }
-
- BucketWriter(long rollInterval, long rollSize, long rollCount, long batchSize,
- Context context, String filePath, String fileName, String inUsePrefix,
- String inUseSuffix, String fileSuffix, CompressionCodec codeC,
- CompressionType compType, HDFSWriter writer,
- ScheduledExecutorService timedRollerPool, PrivilegedExecutor proxyUser,
- SinkCounter sinkCounter, int idleTimeout, WriterCallback onCloseCallback,
- String onCloseCallbackPath, long callTimeout,
- ExecutorService callTimeoutPool, long retryInterval,
- int maxRetries, Clock clock) {
- this.rollInterval = rollInterval;
- this.rollSize = rollSize;
- this.rollCount = rollCount;
- this.batchSize = batchSize;
- this.filePath = filePath;
- this.fileName = fileName;
- this.inUsePrefix = inUsePrefix;
- this.inUseSuffix = inUseSuffix;
- this.fileSuffix = fileSuffix;
- this.codeC = codeC;
- this.compType = compType;
- this.writer = writer;
- this.timedRollerPool = timedRollerPool;
- this.proxyUser = proxyUser;
- this.sinkCounter = sinkCounter;
- this.idleTimeout = idleTimeout;
- this.onCloseCallback = onCloseCallback;
- this.onCloseCallbackPath = onCloseCallbackPath;
- this.callTimeout = callTimeout;
- this.callTimeoutPool = callTimeoutPool;
- fileExtensionCounter = new AtomicLong(clock.currentTimeMillis());
-
- this.retryInterval = retryInterval;
- this.maxRetries = maxRetries;
- isOpen = false;
- isUnderReplicated = false;
- this.writer.configure(context);
- }
-
- @VisibleForTesting
- void setFileSystem(FileSystem fs) {
- this.fileSystem = fs;
- mockFsInjected = true;
- }
-
- /**
- * Clear the class counters
- */
- private void resetCounters() {
- eventCounter = 0;
- processSize = 0;
- batchCounter = 0;
- }
-
- private Method getRefIsClosed() {
- try {
- return fileSystem.getClass().getMethod("isFileClosed",
- Path.class);
- } catch (Exception e) {
- LOG.info("isFileClosed() is not available in the version of the " +
- "distributed filesystem being used. " +
- "Flume will not attempt to re-close files if the close fails " +
- "on the first attempt");
- return null;
- }
- }
-
- private Boolean isFileClosed(FileSystem fs, Path tmpFilePath) throws Exception {
- return (Boolean)(isClosedMethod.invoke(fs, tmpFilePath));
- }
-
- /**
- * open() is called by append()
- * @throws IOException
- * @throws InterruptedException
- */
- private void open() throws IOException, InterruptedException {
- if ((filePath == null) || (writer == null)) {
- throw new IOException("Invalid file settings");
- }
-
- final Configuration config = new Configuration();
- // disable FileSystem JVM shutdown hook
- config.setBoolean("fs.automatic.close", false);
-
- // Hadoop is not thread safe when doing certain RPC operations,
- // including getFileSystem(), when running under Kerberos.
- // open() must be called by one thread at a time in the JVM.
- // NOTE: tried synchronizing on the underlying Kerberos principal previously
- // which caused deadlocks. See FLUME-1231.
- synchronized (staticLock) {
- checkAndThrowInterruptedException();
-
- try {
- long counter = fileExtensionCounter.incrementAndGet();
-
- String fullFileName = fileName + "." + counter;
-
- if (fileSuffix != null && fileSuffix.length() > 0) {
- fullFileName += fileSuffix;
- } else if (codeC != null) {
- fullFileName += codeC.getDefaultExtension();
- }
-
- bucketPath = filePath + "/" + inUsePrefix
- + fullFileName + inUseSuffix;
- targetPath = filePath + "/" + fullFileName;
-
- LOG.info("Creating " + bucketPath);
- callWithTimeout(new CallRunner<Void>() {
- @Override
- public Void call() throws Exception {
- if (codeC == null) {
- // Need to get reference to FS using above config before underlying
- // writer does in order to avoid shutdown hook &
- // IllegalStateExceptions
- if (!mockFsInjected) {
- fileSystem = new Path(bucketPath).getFileSystem(config);
- }
- writer.open(bucketPath);
- } else {
- // need to get reference to FS before writer does to
- // avoid shutdown hook
- if (!mockFsInjected) {
- fileSystem = new Path(bucketPath).getFileSystem(config);
- }
- writer.open(bucketPath, codeC, compType);
- }
- return null;
- }
- });
- } catch (Exception ex) {
- sinkCounter.incrementConnectionFailedCount();
- if (ex instanceof IOException) {
- throw (IOException) ex;
- } else {
- throw Throwables.propagate(ex);
- }
- }
- }
- isClosedMethod = getRefIsClosed();
- sinkCounter.incrementConnectionCreatedCount();
- resetCounters();
-
- // if time-based rolling is enabled, schedule the roll
- if (rollInterval > 0) {
- Callable<Void> action = new Callable<Void>() {
- public Void call() throws Exception {
- LOG.debug("Rolling file ({}): Roll scheduled after {} sec elapsed.",
- bucketPath, rollInterval);
- try {
- // Roll the file and remove reference from sfWriters map.
- close(true);
- } catch (Throwable t) {
- LOG.error("Unexpected error", t);
- }
- return null;
- }
- };
- timedRollFuture = timedRollerPool.schedule(action, rollInterval,
- TimeUnit.SECONDS);
- }
-
- isOpen = true;
- }
-
- /**
- * Close the file handle and rename the temp file to the permanent filename.
- * Safe to call multiple times. Logs HDFSWriter.close() exceptions. This
- * method will not cause the bucket writer to be dereferenced from the HDFS
- * sink that owns it. This method should be used only when size or count
- * based rolling closes this file.
- */
- public void close() throws InterruptedException {
- close(false);
- }
-
- private CallRunner<Void> createCloseCallRunner() {
- return new CallRunner<Void>() {
- @Override
- public Void call() throws Exception {
- writer.close(); // could block
- return null;
- }
- };
- }
-
- private class CloseHandler implements Callable<Void> {
- private final String path = bucketPath;
- private int closeTries = 0;
-
- @Override
- public Void call() throws Exception {
- close(false);
- return null;
- }
-
- /**
- * Tries to close the writer. Repeats the close if the maximum number
- * of retries is not reached or an immediate close is not reuqested.
- * If all close attempts were unsuccessful we try to recover the lease.
- * @param immediate An immediate close is required
- */
- public void close(boolean immediate) {
- closeTries++;
- boolean shouldRetry = closeTries < maxRetries && !immediate;
- try {
- callWithTimeout(createCloseCallRunner());
- sinkCounter.incrementConnectionClosedCount();
- } catch (InterruptedException | IOException e) {
- LOG.warn("Closing file: " + path + " failed. Will " +
- "retry again in " + retryInterval + " seconds.", e);
- if (timedRollerPool != null && !timedRollerPool.isTerminated()) {
- if (shouldRetry) {
- timedRollerPool.schedule(this, retryInterval, TimeUnit.SECONDS);
- }
- } else {
- LOG.warn("Cannot retry close any more timedRollerPool is null or terminated");
- }
- if (!shouldRetry) {
- LOG.warn("Unsuccessfully attempted to close " + path + " " +
- maxRetries + " times. Initializing lease recovery.");
- sinkCounter.incrementConnectionFailedCount();
- recoverLease();
- }
- }
- }
- }
-
- private class ScheduledRenameCallable implements Callable<Void> {
- private final String path = bucketPath;
- private final String finalPath = targetPath;
- private FileSystem fs = fileSystem;
- private int renameTries = 1; // one attempt is already done
-
- @Override
- public Void call() throws Exception {
- if (renameTries >= maxRetries) {
- LOG.warn("Unsuccessfully attempted to rename " + path + " " +
- maxRetries + " times. File may still be open.");
- return null;
- }
- renameTries++;
- try {
- renameBucket(path, finalPath, fs);
- } catch (Exception e) {
- LOG.warn("Renaming file: " + path + " failed. Will " +
- "retry again in " + retryInterval + " seconds.", e);
- timedRollerPool.schedule(this, retryInterval, TimeUnit.SECONDS);
- return null;
- }
- return null;
- }
- }
-
- /**
- * Tries to start the lease recovery process for the current bucketPath
- * if the fileSystem is DistributedFileSystem.
- * Catches and logs the IOException.
- */
- private synchronized void recoverLease() {
- if (bucketPath != null && fileSystem instanceof DistributedFileSystem) {
- try {
- LOG.debug("Starting lease recovery for {}", bucketPath);
- ((DistributedFileSystem) fileSystem).recoverLease(new Path(bucketPath));
- } catch (IOException ex) {
- LOG.warn("Lease recovery failed for {}", bucketPath, ex);
- }
- }
- }
-
- public void close(boolean callCloseCallback) throws InterruptedException {
- close(callCloseCallback, false);
- }
-
- /**
- * Close the file handle and rename the temp file to the permanent filename.
- * Safe to call multiple times. Logs HDFSWriter.close() exceptions.
- */
- public void close(boolean callCloseCallback, boolean immediate)
- throws InterruptedException {
- if (callCloseCallback) {
- if (closed.compareAndSet(false, true)) {
- runCloseAction(); //remove from the cache as soon as possible
- } else {
- LOG.warn("This bucketWriter is already closing or closed.");
- }
- }
- doClose(immediate);
- }
-
- private synchronized void doClose(boolean immediate)
- throws InterruptedException {
- checkAndThrowInterruptedException();
- try {
- flush();
- } catch (IOException e) {
- LOG.warn("pre-close flush failed", e);
- }
-
- LOG.info("Closing {}", bucketPath);
- if (isOpen) {
- new CloseHandler().close(immediate);
- isOpen = false;
- } else {
- LOG.info("HDFSWriter is already closed: {}", bucketPath);
- }
-
- // NOTE: timed rolls go through this codepath as well as other roll types
- if (timedRollFuture != null && !timedRollFuture.isDone()) {
- timedRollFuture.cancel(false); // do not cancel myself if running!
- timedRollFuture = null;
- }
-
- if (idleFuture != null && !idleFuture.isDone()) {
- idleFuture.cancel(false); // do not cancel myself if running!
- idleFuture = null;
- }
-
- if (bucketPath != null && fileSystem != null) {
- // could block or throw IOException
- try {
- renameBucket(bucketPath, targetPath, fileSystem);
- } catch (Exception e) {
- LOG.warn("failed to rename() file (" + bucketPath +
- "). Exception follows.", e);
- sinkCounter.incrementConnectionFailedCount();
- final Callable<Void> scheduledRename = new ScheduledRenameCallable();
- timedRollerPool.schedule(scheduledRename, retryInterval, TimeUnit.SECONDS);
- }
- }
- }
-
- /**
- * flush the data
- * @throws IOException
- * @throws InterruptedException
- */
- public synchronized void flush() throws IOException, InterruptedException {
- checkAndThrowInterruptedException();
- if (!isBatchComplete()) {
- doFlush();
-
- if (idleTimeout > 0) {
- // if the future exists and couldn't be cancelled, that would mean it has already run
- // or been cancelled
- if (idleFuture == null || idleFuture.cancel(false)) {
- Callable<Void> idleAction = new Callable<Void>() {
- public Void call() throws Exception {
- LOG.info("Closing idle bucketWriter {} at {}", bucketPath,
- System.currentTimeMillis());
- if (isOpen) {
- close(true);
- }
- return null;
- }
- };
- idleFuture = timedRollerPool.schedule(idleAction, idleTimeout,
- TimeUnit.SECONDS);
- }
- }
- }
- }
-
- private void runCloseAction() {
- try {
- if (onCloseCallback != null) {
- onCloseCallback.run(onCloseCallbackPath);
- }
- } catch (Throwable t) {
- LOG.error("Unexpected error", t);
- }
- }
-
- /**
- * doFlush() must only be called by flush()
- * @throws IOException
- */
- private void doFlush() throws IOException, InterruptedException {
- callWithTimeout(new CallRunner<Void>() {
- @Override
- public Void call() throws Exception {
- writer.sync(); // could block
- return null;
- }
- });
- batchCounter = 0;
- }
-
- /**
- * Open file handles, write data, update stats, handle file rolling and
- * batching / flushing. <br />
- * If the write fails, the file is implicitly closed and then the IOException
- * is rethrown. <br />
- * We rotate before append, and not after, so that the active file rolling
- * mechanism will never roll an empty file. This also ensures that the file
- * creation time reflects when the first event was written.
- *
- * @throws IOException
- * @throws InterruptedException
- */
- public synchronized void append(final Event event)
- throws IOException, InterruptedException {
- checkAndThrowInterruptedException();
- // If idleFuture is not null, cancel it before we move forward to avoid a
- // close call in the middle of the append.
- if (idleFuture != null) {
- idleFuture.cancel(false);
- // There is still a small race condition - if the idleFuture is already
- // running, interrupting it can cause HDFS close operation to throw -
- // so we cannot interrupt it while running. If the future could not be
- // cancelled, it is already running - wait for it to finish before
- // attempting to write.
- if (!idleFuture.isDone()) {
- try {
- idleFuture.get(callTimeout, TimeUnit.MILLISECONDS);
- } catch (TimeoutException ex) {
- LOG.warn("Timeout while trying to cancel closing of idle file. Idle" +
- " file close may have failed", ex);
- } catch (Exception ex) {
- LOG.warn("Error while trying to cancel closing of idle file. ", ex);
- }
- }
- idleFuture = null;
- }
-
- // If the bucket writer was closed due to roll timeout or idle timeout,
- // force a new bucket writer to be created. Roll count and roll size will
- // just reuse this one
- if (!isOpen) {
- if (closed.get()) {
- throw new BucketClosedException("This bucket writer was closed and " +
- "this handle is thus no longer valid");
- }
- open();
- }
-
- // check if it's time to rotate the file
- if (shouldRotate()) {
- boolean doRotate = true;
-
- if (isUnderReplicated) {
- if (maxConsecUnderReplRotations > 0 &&
- consecutiveUnderReplRotateCount >= maxConsecUnderReplRotations) {
- doRotate = false;
- if (consecutiveUnderReplRotateCount == maxConsecUnderReplRotations) {
- LOG.error("Hit max consecutive under-replication rotations ({}); " +
- "will not continue rolling files under this path due to " +
- "under-replication", maxConsecUnderReplRotations);
- }
- } else {
- LOG.warn("Block Under-replication detected. Rotating file.");
- }
- consecutiveUnderReplRotateCount++;
- } else {
- consecutiveUnderReplRotateCount = 0;
- }
-
- if (doRotate) {
- close();
- open();
- }
- }
-
- // write the event
- try {
- sinkCounter.incrementEventDrainAttemptCount();
- callWithTimeout(new CallRunner<Void>() {
- @Override
- public Void call() throws Exception {
- writer.append(event); // could block
- return null;
- }
- });
- } catch (IOException e) {
- LOG.warn("Caught IOException writing to HDFSWriter ({}). Closing file (" +
- bucketPath + ") and rethrowing exception.",
- e.getMessage());
- close(true);
- throw e;
- }
-
- // update statistics
- processSize += event.getBody().length;
- eventCounter++;
- batchCounter++;
-
- if (batchCounter == batchSize) {
- flush();
- }
- }
-
- /**
- * check if time to rotate the file
- */
- private boolean shouldRotate() {
- boolean doRotate = false;
-
- if (writer.isUnderReplicated()) {
- this.isUnderReplicated = true;
- doRotate = true;
- } else {
- this.isUnderReplicated = false;
- }
-
- if ((rollCount > 0) && (rollCount <= eventCounter)) {
- LOG.debug("rolling: rollCount: {}, events: {}", rollCount, eventCounter);
- doRotate = true;
- }
-
- if ((rollSize > 0) && (rollSize <= processSize)) {
- LOG.debug("rolling: rollSize: {}, bytes: {}", rollSize, processSize);
- doRotate = true;
- }
-
- return doRotate;
- }
-
- /**
- * Rename bucketPath file from .tmp to permanent location.
- */
- // When this bucket writer is rolled based on rollCount or
- // rollSize, the same instance is reused for the new file. But if
- // the previous file was not closed/renamed,
- // the bucket writer fields no longer point to it and hence need
- // to be passed in from the thread attempting to close it. Even
- // when the bucket writer is closed due to close timeout,
- // this method can get called from the scheduled thread so the
- // file gets closed later - so an implicit reference to this
- // bucket writer would still be alive in the Callable instance.
- private void renameBucket(String bucketPath, String targetPath, final FileSystem fs)
- throws IOException, InterruptedException {
- if (bucketPath.equals(targetPath)) {
- return;
- }
-
- final Path srcPath = new Path(bucketPath);
- final Path dstPath = new Path(targetPath);
-
- callWithTimeout(new CallRunner<Void>() {
- @Override
- public Void call() throws Exception {
- if (fs.exists(srcPath)) { // could block
- LOG.info("Renaming " + srcPath + " to " + dstPath);
- renameTries.incrementAndGet();
- fs.rename(srcPath, dstPath); // could block
- }
- return null;
- }
- });
- }
-
- @Override
- public String toString() {
- return "[ " + this.getClass().getSimpleName() + " targetPath = " + targetPath +
- ", bucketPath = " + bucketPath + " ]";
- }
-
- private boolean isBatchComplete() {
- return (batchCounter == 0);
- }
-
- /**
- * This method if the current thread has been interrupted and throws an
- * exception.
- * @throws InterruptedException
- */
- private static void checkAndThrowInterruptedException()
- throws InterruptedException {
- if (Thread.currentThread().interrupted()) {
- throw new InterruptedException("Timed out before HDFS call was made. "
- + "Your hdfs.callTimeout might be set too low or HDFS calls are "
- + "taking too long.");
- }
- }
-
- /**
- * Execute the callable on a separate thread and wait for the completion
- * for the specified amount of time in milliseconds. In case of timeout
- * cancel the callable and throw an IOException
- */
- private <T> T callWithTimeout(final CallRunner<T> callRunner)
- throws IOException, InterruptedException {
- Future<T> future = callTimeoutPool.submit(new Callable<T>() {
- @Override
- public T call() throws Exception {
- return proxyUser.execute(new PrivilegedExceptionAction<T>() {
- @Override
- public T run() throws Exception {
- return callRunner.call();
- }
- });
- }
- });
- try {
- if (callTimeout > 0) {
- return future.get(callTimeout, TimeUnit.MILLISECONDS);
- } else {
- return future.get();
- }
- } catch (TimeoutException eT) {
- future.cancel(true);
- sinkCounter.incrementConnectionFailedCount();
- throw new IOException("Callable timed out after " +
- callTimeout + " ms" + " on file: " + bucketPath, eT);
- } catch (ExecutionException e1) {
- sinkCounter.incrementConnectionFailedCount();
- Throwable cause = e1.getCause();
- if (cause instanceof IOException) {
- throw (IOException) cause;
- } else if (cause instanceof InterruptedException) {
- throw (InterruptedException) cause;
- } else if (cause instanceof RuntimeException) {
- throw (RuntimeException) cause;
- } else if (cause instanceof Error) {
- throw (Error)cause;
- } else {
- throw new RuntimeException(e1);
- }
- } catch (CancellationException ce) {
- throw new InterruptedException(
- "Blocked callable interrupted by rotation event");
- } catch (InterruptedException ex) {
- LOG.warn("Unexpected Exception " + ex.getMessage(), ex);
- throw ex;
- }
- }
-
- /**
- * Simple interface whose <tt>call</tt> method is called by
- * {#callWithTimeout} in a new thread inside a
- * {@linkplain java.security.PrivilegedExceptionAction#run()} call.
- * @param <T>
- */
- private interface CallRunner<T> {
- T call() throws Exception;
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSCompressedDataStream.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSCompressedDataStream.java
deleted file mode 100644
index 80b7cb4c3..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSCompressedDataStream.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.serialization.EventSerializer;
-import org.apache.flume.serialization.EventSerializerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CodecPool;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionOutputStream;
-import org.apache.hadoop.io.compress.Compressor;
-import org.apache.hadoop.io.compress.DefaultCodec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HDFSCompressedDataStream extends AbstractHDFSWriter {
-
- private static final Logger logger =
- LoggerFactory.getLogger(HDFSCompressedDataStream.class);
-
- private FSDataOutputStream fsOut;
- private CompressionOutputStream cmpOut;
- private boolean isFinished = false;
-
- private String serializerType;
- private Context serializerContext;
- private EventSerializer serializer;
- private boolean useRawLocalFileSystem;
- private Compressor compressor;
-
- @Override
- public void configure(Context context) {
- super.configure(context);
-
- serializerType = context.getString("serializer", "TEXT");
- useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
- false);
- serializerContext = new Context(
- context.getSubProperties(EventSerializer.CTX_PREFIX));
- logger.info("Serializer = " + serializerType + ", UseRawLocalFileSystem = "
- + useRawLocalFileSystem);
- }
-
- @Override
- public void open(String filePath) throws IOException {
- DefaultCodec defCodec = new DefaultCodec();
- CompressionType cType = CompressionType.BLOCK;
- open(filePath, defCodec, cType);
- }
-
- @Override
- public void open(String filePath, CompressionCodec codec,
- CompressionType cType) throws IOException {
- Configuration conf = new Configuration();
- Path dstPath = new Path(filePath);
- FileSystem hdfs = dstPath.getFileSystem(conf);
- if (useRawLocalFileSystem) {
- if (hdfs instanceof LocalFileSystem) {
- hdfs = ((LocalFileSystem)hdfs).getRaw();
- } else {
- logger.warn("useRawLocalFileSystem is set to true but file system " +
- "is not of type LocalFileSystem: " + hdfs.getClass().getName());
- }
- }
- boolean appending = false;
- if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) {
- fsOut = hdfs.append(dstPath);
- appending = true;
- } else {
- fsOut = hdfs.create(dstPath);
- }
- if (compressor == null) {
- compressor = CodecPool.getCompressor(codec, conf);
- }
- cmpOut = codec.createOutputStream(fsOut, compressor);
- serializer = EventSerializerFactory.getInstance(serializerType,
- serializerContext, cmpOut);
- if (appending && !serializer.supportsReopen()) {
- cmpOut.close();
- serializer = null;
- throw new IOException("serializer (" + serializerType
- + ") does not support append");
- }
-
- registerCurrentStream(fsOut, hdfs, dstPath);
-
- if (appending) {
- serializer.afterReopen();
- } else {
- serializer.afterCreate();
- }
- isFinished = false;
- }
-
- @Override
- public void append(Event e) throws IOException {
- if (isFinished) {
- cmpOut.resetState();
- isFinished = false;
- }
- serializer.write(e);
- }
-
- @Override
- public void sync() throws IOException {
- // We must use finish() and resetState() here -- flush() is apparently not
- // supported by the compressed output streams (it's a no-op).
- // Also, since resetState() writes headers, avoid calling it without an
- // additional write/append operation.
- // Note: There are bugs in Hadoop & JDK w/ pure-java gzip; see HADOOP-8522.
- serializer.flush();
- if (!isFinished) {
- cmpOut.finish();
- isFinished = true;
- }
- fsOut.flush();
- hflushOrSync(this.fsOut);
- }
-
- @Override
- public void close() throws IOException {
- serializer.flush();
- serializer.beforeClose();
- if (!isFinished) {
- cmpOut.finish();
- isFinished = true;
- }
- fsOut.flush();
- hflushOrSync(fsOut);
- cmpOut.close();
- if (compressor != null) {
- CodecPool.returnCompressor(compressor);
- compressor = null;
- }
- unregisterCurrentStream();
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSDataStream.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSDataStream.java
deleted file mode 100644
index c4ad9192d..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSDataStream.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.serialization.EventSerializer;
-import org.apache.flume.serialization.EventSerializerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HDFSDataStream extends AbstractHDFSWriter {
-
- private static final Logger logger = LoggerFactory.getLogger(HDFSDataStream.class);
-
- private FSDataOutputStream outStream;
- private String serializerType;
- private Context serializerContext;
- private EventSerializer serializer;
- private boolean useRawLocalFileSystem;
-
- @Override
- public void configure(Context context) {
- super.configure(context);
-
- serializerType = context.getString("serializer", "TEXT");
- useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
- false);
- serializerContext =
- new Context(context.getSubProperties(EventSerializer.CTX_PREFIX));
- logger.info("Serializer = " + serializerType + ", UseRawLocalFileSystem = "
- + useRawLocalFileSystem);
- }
-
- @VisibleForTesting
- protected FileSystem getDfs(Configuration conf, Path dstPath) throws IOException {
- return dstPath.getFileSystem(conf);
- }
-
- protected void doOpen(Configuration conf, Path dstPath, FileSystem hdfs) throws IOException {
- if (useRawLocalFileSystem) {
- if (hdfs instanceof LocalFileSystem) {
- hdfs = ((LocalFileSystem)hdfs).getRaw();
- } else {
- logger.warn("useRawLocalFileSystem is set to true but file system " +
- "is not of type LocalFileSystem: " + hdfs.getClass().getName());
- }
- }
-
- boolean appending = false;
- if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) {
- outStream = hdfs.append(dstPath);
- appending = true;
- } else {
- outStream = hdfs.create(dstPath);
- }
-
- serializer = EventSerializerFactory.getInstance(
- serializerType, serializerContext, outStream);
- if (appending && !serializer.supportsReopen()) {
- outStream.close();
- serializer = null;
- throw new IOException("serializer (" + serializerType +
- ") does not support append");
- }
-
- // must call superclass to check for replication issues
- registerCurrentStream(outStream, hdfs, dstPath);
-
- if (appending) {
- serializer.afterReopen();
- } else {
- serializer.afterCreate();
- }
- }
-
- @Override
- public void open(String filePath) throws IOException {
- Configuration conf = new Configuration();
- Path dstPath = new Path(filePath);
- FileSystem hdfs = getDfs(conf, dstPath);
- doOpen(conf, dstPath, hdfs);
- }
-
- @Override
- public void open(String filePath, CompressionCodec codec,
- CompressionType cType) throws IOException {
- open(filePath);
- }
-
- @Override
- public void append(Event e) throws IOException {
- serializer.write(e);
- }
-
- @Override
- public void sync() throws IOException {
- serializer.flush();
- outStream.flush();
- hflushOrSync(outStream);
- }
-
- @Override
- public void close() throws IOException {
- serializer.flush();
- serializer.beforeClose();
- outStream.flush();
- hflushOrSync(outStream);
- outStream.close();
-
- unregisterCurrentStream();
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSEventSink.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSEventSink.java
deleted file mode 100644
index 9e64eb532..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSEventSink.java
+++ /dev/null
@@ -1,577 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.Map.Entry;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.flume.Channel;
-import org.apache.flume.Clock;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.SystemClock;
-import org.apache.flume.Transaction;
-import org.apache.flume.auth.FlumeAuthenticationUtil;
-import org.apache.flume.auth.PrivilegedExecutor;
-import org.apache.flume.conf.BatchSizeSupported;
-import org.apache.flume.conf.Configurable;
-import org.apache.flume.formatter.output.BucketPath;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.flume.sink.AbstractSink;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-public class HDFSEventSink extends AbstractSink implements Configurable, BatchSizeSupported {
- public interface WriterCallback {
- public void run(String filePath);
- }
-
- private static final Logger LOG = LoggerFactory.getLogger(HDFSEventSink.class);
-
- private static String DIRECTORY_DELIMITER = System.getProperty("file.separator");
-
- private static final long defaultRollInterval = 30;
- private static final long defaultRollSize = 1024;
- private static final long defaultRollCount = 10;
- private static final String defaultFileName = "FlumeData";
- private static final String defaultSuffix = "";
- private static final String defaultInUsePrefix = "";
- private static final String defaultInUseSuffix = ".tmp";
- private static final long defaultBatchSize = 100;
- private static final String defaultFileType = HDFSWriterFactory.SequenceFileType;
- private static final int defaultMaxOpenFiles = 5000;
- // Time between close retries, in seconds
- private static final long defaultRetryInterval = 180;
- // Retry forever.
- private static final int defaultTryCount = Integer.MAX_VALUE;
-
- public static final String IN_USE_SUFFIX_PARAM_NAME = "hdfs.inUseSuffix";
-
- /**
- * Default length of time we wait for blocking BucketWriter calls
- * before timing out the operation. Intended to prevent server hangs.
- */
- private static final long defaultCallTimeout = 30000;
- /**
- * Default number of threads available for tasks
- * such as append/open/close/flush with hdfs.
- * These tasks are done in a separate thread in
- * the case that they take too long. In which
- * case we create a new file and move on.
- */
- private static final int defaultThreadPoolSize = 10;
- private static final int defaultRollTimerPoolSize = 1;
-
- private final HDFSWriterFactory writerFactory;
- private WriterLinkedHashMap sfWriters;
-
- private long rollInterval;
- private long rollSize;
- private long rollCount;
- private long batchSize;
- private int threadsPoolSize;
- private int rollTimerPoolSize;
- private CompressionCodec codeC;
- private CompressionType compType;
- private String fileType;
- private String filePath;
- private String fileName;
- private String suffix;
- private String inUsePrefix;
- private String inUseSuffix;
- private TimeZone timeZone;
- private int maxOpenFiles;
- private ExecutorService callTimeoutPool;
- private ScheduledExecutorService timedRollerPool;
-
- private boolean needRounding = false;
- private int roundUnit = Calendar.SECOND;
- private int roundValue = 1;
- private boolean useLocalTime = false;
-
- private long callTimeout;
- private Context context;
- private SinkCounter sinkCounter;
-
- private volatile int idleTimeout;
- private Clock clock;
- private FileSystem mockFs;
- private HDFSWriter mockWriter;
- private final Object sfWritersLock = new Object();
- private long retryInterval;
- private int tryCount;
- private PrivilegedExecutor privExecutor;
-
- /*
- * Extended Java LinkedHashMap for open file handle LRU queue.
- * We want to clear the oldest file handle if there are too many open ones.
- */
- private static class WriterLinkedHashMap
- extends LinkedHashMap<String, BucketWriter> {
-
- private final int maxOpenFiles;
-
- public WriterLinkedHashMap(int maxOpenFiles) {
- super(16, 0.75f, true); // stock initial capacity/load, access ordering
- this.maxOpenFiles = maxOpenFiles;
- }
-
- @Override
- protected boolean removeEldestEntry(Entry<String, BucketWriter> eldest) {
- if (size() > maxOpenFiles) {
- // If we have more that max open files, then close the last one and
- // return true
- try {
- eldest.getValue().close();
- } catch (InterruptedException e) {
- LOG.warn(eldest.getKey().toString(), e);
- Thread.currentThread().interrupt();
- }
- return true;
- } else {
- return false;
- }
- }
- }
-
- public HDFSEventSink() {
- this(new HDFSWriterFactory());
- }
-
- public HDFSEventSink(HDFSWriterFactory writerFactory) {
- this.writerFactory = writerFactory;
- }
-
- @VisibleForTesting
- Map<String, BucketWriter> getSfWriters() {
- return sfWriters;
- }
-
- // read configuration and setup thresholds
- @Override
- public void configure(Context context) {
- this.context = context;
-
- filePath = Preconditions.checkNotNull(
- context.getString("hdfs.path"), "hdfs.path is required");
- fileName = context.getString("hdfs.filePrefix", defaultFileName);
- this.suffix = context.getString("hdfs.fileSuffix", defaultSuffix);
- inUsePrefix = context.getString("hdfs.inUsePrefix", defaultInUsePrefix);
- boolean emptyInUseSuffix = context.getBoolean("hdfs.emptyInUseSuffix", false);
- if (emptyInUseSuffix) {
- inUseSuffix = "";
- String tmpInUseSuffix = context.getString(IN_USE_SUFFIX_PARAM_NAME);
- if (tmpInUseSuffix != null) {
- LOG.warn("Ignoring parameter " + IN_USE_SUFFIX_PARAM_NAME + " for hdfs sink: " + getName());
- }
- } else {
- inUseSuffix = context.getString(IN_USE_SUFFIX_PARAM_NAME, defaultInUseSuffix);
- }
- String tzName = context.getString("hdfs.timeZone");
- timeZone = tzName == null ? null : TimeZone.getTimeZone(tzName);
- rollInterval = context.getLong("hdfs.rollInterval", defaultRollInterval);
- rollSize = context.getLong("hdfs.rollSize", defaultRollSize);
- rollCount = context.getLong("hdfs.rollCount", defaultRollCount);
- batchSize = context.getLong("hdfs.batchSize", defaultBatchSize);
- idleTimeout = context.getInteger("hdfs.idleTimeout", 0);
- String codecName = context.getString("hdfs.codeC");
- fileType = context.getString("hdfs.fileType", defaultFileType);
- maxOpenFiles = context.getInteger("hdfs.maxOpenFiles", defaultMaxOpenFiles);
- callTimeout = context.getLong("hdfs.callTimeout", defaultCallTimeout);
- threadsPoolSize = context.getInteger("hdfs.threadsPoolSize",
- defaultThreadPoolSize);
- rollTimerPoolSize = context.getInteger("hdfs.rollTimerPoolSize",
- defaultRollTimerPoolSize);
- String kerbConfPrincipal = context.getString("hdfs.kerberosPrincipal");
- String kerbKeytab = context.getString("hdfs.kerberosKeytab");
- String proxyUser = context.getString("hdfs.proxyUser");
- tryCount = context.getInteger("hdfs.closeTries", defaultTryCount);
- if (tryCount <= 0) {
- LOG.warn("Retry count value : " + tryCount + " is not " +
- "valid. The sink will try to close the file until the file " +
- "is eventually closed.");
- tryCount = defaultTryCount;
- }
- retryInterval = context.getLong("hdfs.retryInterval", defaultRetryInterval);
- if (retryInterval <= 0) {
- LOG.warn("Retry Interval value: " + retryInterval + " is not " +
- "valid. If the first close of a file fails, " +
- "it may remain open and will not be renamed.");
- tryCount = 1;
- }
-
- Preconditions.checkArgument(batchSize > 0, "batchSize must be greater than 0");
- if (codecName == null) {
- codeC = null;
- compType = CompressionType.NONE;
- } else {
- codeC = getCodec(codecName);
- // TODO : set proper compression type
- compType = CompressionType.BLOCK;
- }
-
- // Do not allow user to set fileType DataStream with codeC together
- // To prevent output file with compress extension (like .snappy)
- if (fileType.equalsIgnoreCase(HDFSWriterFactory.DataStreamType) && codecName != null) {
- throw new IllegalArgumentException("fileType: " + fileType +
- " which does NOT support compressed output. Please don't set codeC" +
- " or change the fileType if compressed output is desired.");
- }
-
- if (fileType.equalsIgnoreCase(HDFSWriterFactory.CompStreamType)) {
- Preconditions.checkNotNull(codeC, "It's essential to set compress codec"
- + " when fileType is: " + fileType);
- }
-
- // get the appropriate executor
- this.privExecutor = FlumeAuthenticationUtil.getAuthenticator(
- kerbConfPrincipal, kerbKeytab).proxyAs(proxyUser);
-
- needRounding = context.getBoolean("hdfs.round", false);
-
- if (needRounding) {
- String unit = context.getString("hdfs.roundUnit", "second");
- if (unit.equalsIgnoreCase("hour")) {
- this.roundUnit = Calendar.HOUR_OF_DAY;
- } else if (unit.equalsIgnoreCase("minute")) {
- this.roundUnit = Calendar.MINUTE;
- } else if (unit.equalsIgnoreCase("second")) {
- this.roundUnit = Calendar.SECOND;
- } else {
- LOG.warn("Rounding unit is not valid, please set one of" +
- "minute, hour, or second. Rounding will be disabled");
- needRounding = false;
- }
- this.roundValue = context.getInteger("hdfs.roundValue", 1);
- if (roundUnit == Calendar.SECOND || roundUnit == Calendar.MINUTE) {
- Preconditions.checkArgument(roundValue > 0 && roundValue <= 60,
- "Round value" +
- "must be > 0 and <= 60");
- } else if (roundUnit == Calendar.HOUR_OF_DAY) {
- Preconditions.checkArgument(roundValue > 0 && roundValue <= 24,
- "Round value" +
- "must be > 0 and <= 24");
- }
- }
-
- this.useLocalTime = context.getBoolean("hdfs.useLocalTimeStamp", false);
- if (useLocalTime) {
- clock = new SystemClock();
- }
-
- if (sinkCounter == null) {
- sinkCounter = new SinkCounter(getName());
- }
- }
-
- private static boolean codecMatches(Class<? extends CompressionCodec> cls, String codecName) {
- String simpleName = cls.getSimpleName();
- if (cls.getName().equals(codecName) || simpleName.equalsIgnoreCase(codecName)) {
- return true;
- }
- if (simpleName.endsWith("Codec")) {
- String prefix = simpleName.substring(0, simpleName.length() - "Codec".length());
- if (prefix.equalsIgnoreCase(codecName)) {
- return true;
- }
- }
- return false;
- }
-
- @VisibleForTesting
- static CompressionCodec getCodec(String codecName) {
- Configuration conf = new Configuration();
- List<Class<? extends CompressionCodec>> codecs = CompressionCodecFactory.getCodecClasses(conf);
- // Wish we could base this on DefaultCodec but appears not all codec's
- // extend DefaultCodec(Lzo)
- CompressionCodec codec = null;
- ArrayList<String> codecStrs = new ArrayList<String>();
- codecStrs.add("None");
- for (Class<? extends CompressionCodec> cls : codecs) {
- codecStrs.add(cls.getSimpleName());
- if (codecMatches(cls, codecName)) {
- try {
- codec = cls.newInstance();
- } catch (InstantiationException e) {
- LOG.error("Unable to instantiate " + cls + " class");
- } catch (IllegalAccessException e) {
- LOG.error("Unable to access " + cls + " class");
- }
- }
- }
-
- if (codec == null) {
- if (!codecName.equalsIgnoreCase("None")) {
- throw new IllegalArgumentException("Unsupported compression codec "
- + codecName + ". Please choose from: " + codecStrs);
- }
- } else if (codec instanceof org.apache.hadoop.conf.Configurable) {
- // Must check instanceof codec as BZip2Codec doesn't inherit Configurable
- // Must set the configuration for Configurable objects that may or do use
- // native libs
- ((org.apache.hadoop.conf.Configurable) codec).setConf(conf);
- }
- return codec;
- }
-
-
- /**
- * Pull events out of channel and send it to HDFS. Take at most batchSize
- * events per Transaction. Find the corresponding bucket for the event.
- * Ensure the file is open. Serialize the data and write it to the file on
- * HDFS. <br/>
- * This method is not thread safe.
- */
- public Status process() throws EventDeliveryException {
- Channel channel = getChannel();
- Transaction transaction = channel.getTransaction();
- transaction.begin();
- try {
- Set<BucketWriter> writers = new LinkedHashSet<>();
- int txnEventCount = 0;
- for (txnEventCount = 0; txnEventCount < batchSize; txnEventCount++) {
- Event event = channel.take();
- if (event == null) {
- break;
- }
-
- // reconstruct the path name by substituting place holders
- String realPath = BucketPath.escapeString(filePath, event.getHeaders(),
- timeZone, needRounding, roundUnit, roundValue, useLocalTime);
- String realName = BucketPath.escapeString(fileName, event.getHeaders(),
- timeZone, needRounding, roundUnit, roundValue, useLocalTime);
-
- String lookupPath = realPath + DIRECTORY_DELIMITER + realName;
- BucketWriter bucketWriter;
- HDFSWriter hdfsWriter = null;
- // Callback to remove the reference to the bucket writer from the
- // sfWriters map so that all buffers used by the HDFS file
- // handles are garbage collected.
- WriterCallback closeCallback = new WriterCallback() {
- @Override
- public void run(String bucketPath) {
- LOG.info("Writer callback called.");
- synchronized (sfWritersLock) {
- sfWriters.remove(bucketPath);
- }
- }
- };
- synchronized (sfWritersLock) {
- bucketWriter = sfWriters.get(lookupPath);
- // we haven't seen this file yet, so open it and cache the handle
- if (bucketWriter == null) {
- hdfsWriter = writerFactory.getWriter(fileType);
- bucketWriter = initializeBucketWriter(realPath, realName,
- lookupPath, hdfsWriter, closeCallback);
- sfWriters.put(lookupPath, bucketWriter);
- }
- }
-
- // Write the data to HDFS
- try {
- bucketWriter.append(event);
- } catch (BucketClosedException ex) {
- LOG.info("Bucket was closed while trying to append, " +
- "reinitializing bucket and writing event.");
- hdfsWriter = writerFactory.getWriter(fileType);
- bucketWriter = initializeBucketWriter(realPath, realName,
- lookupPath, hdfsWriter, closeCallback);
- synchronized (sfWritersLock) {
- sfWriters.put(lookupPath, bucketWriter);
- }
- bucketWriter.append(event);
- }
-
- // track the buckets getting written in this transaction
- if (!writers.contains(bucketWriter)) {
- writers.add(bucketWriter);
- }
- }
-
- if (txnEventCount == 0) {
- sinkCounter.incrementBatchEmptyCount();
- } else if (txnEventCount == batchSize) {
- sinkCounter.incrementBatchCompleteCount();
- } else {
- sinkCounter.incrementBatchUnderflowCount();
- }
-
- // flush all pending buckets before committing the transaction
- for (BucketWriter bucketWriter : writers) {
- bucketWriter.flush();
- }
-
- transaction.commit();
-
- if (txnEventCount < 1) {
- return Status.BACKOFF;
- } else {
- sinkCounter.addToEventDrainSuccessCount(txnEventCount);
- return Status.READY;
- }
- } catch (IOException eIO) {
- transaction.rollback();
- LOG.warn("HDFS IO error", eIO);
- sinkCounter.incrementEventWriteFail();
- return Status.BACKOFF;
- } catch (Throwable th) {
- transaction.rollback();
- LOG.error("process failed", th);
- sinkCounter.incrementEventWriteOrChannelFail(th);
- if (th instanceof Error) {
- throw (Error) th;
- } else {
- throw new EventDeliveryException(th);
- }
- } finally {
- transaction.close();
- }
- }
-
- @VisibleForTesting
- BucketWriter initializeBucketWriter(String realPath,
- String realName, String lookupPath, HDFSWriter hdfsWriter,
- WriterCallback closeCallback) {
- HDFSWriter actualHdfsWriter = mockFs == null ? hdfsWriter : mockWriter;
- BucketWriter bucketWriter = new BucketWriter(rollInterval,
- rollSize, rollCount,
- batchSize, context, realPath, realName, inUsePrefix, inUseSuffix,
- suffix, codeC, compType, actualHdfsWriter, timedRollerPool,
- privExecutor, sinkCounter, idleTimeout, closeCallback,
- lookupPath, callTimeout, callTimeoutPool, retryInterval,
- tryCount);
- if (mockFs != null) {
- bucketWriter.setFileSystem(mockFs);
- }
- return bucketWriter;
- }
-
- @Override
- public void stop() {
- // do not constrain close() calls with a timeout
- synchronized (sfWritersLock) {
- for (Entry<String, BucketWriter> entry : sfWriters.entrySet()) {
- LOG.info("Closing {}", entry.getKey());
-
- try {
- entry.getValue().close(false, true);
- } catch (Exception ex) {
- LOG.warn("Exception while closing " + entry.getKey() + ". " +
- "Exception follows.", ex);
- if (ex instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- }
- }
- }
-
- // shut down all our thread pools
- ExecutorService[] toShutdown = { callTimeoutPool, timedRollerPool };
- for (ExecutorService execService : toShutdown) {
- execService.shutdown();
- try {
- while (execService.isTerminated() == false) {
- execService.awaitTermination(
- Math.max(defaultCallTimeout, callTimeout), TimeUnit.MILLISECONDS);
- }
- } catch (InterruptedException ex) {
- LOG.warn("shutdown interrupted on " + execService, ex);
- }
- }
-
- callTimeoutPool = null;
- timedRollerPool = null;
-
- synchronized (sfWritersLock) {
- sfWriters.clear();
- sfWriters = null;
- }
- sinkCounter.stop();
- super.stop();
- }
-
- @Override
- public void start() {
- String timeoutName = "hdfs-" + getName() + "-call-runner-%d";
- callTimeoutPool = Executors.newFixedThreadPool(threadsPoolSize,
- new ThreadFactoryBuilder().setNameFormat(timeoutName).build());
-
- String rollerName = "hdfs-" + getName() + "-roll-timer-%d";
- timedRollerPool = Executors.newScheduledThreadPool(rollTimerPoolSize,
- new ThreadFactoryBuilder().setNameFormat(rollerName).build());
-
- this.sfWriters = new WriterLinkedHashMap(maxOpenFiles);
- sinkCounter.start();
- super.start();
- }
-
- @Override
- public String toString() {
- return "{ Sink type:" + getClass().getSimpleName() + ", name:" + getName() +
- " }";
- }
-
- @VisibleForTesting
- void setBucketClock(Clock clock) {
- BucketPath.setClock(clock);
- }
-
- @VisibleForTesting
- void setMockFs(FileSystem mockFs) {
- this.mockFs = mockFs;
- }
-
- @VisibleForTesting
- void setMockWriter(HDFSWriter writer) {
- this.mockWriter = writer;
- }
-
- @VisibleForTesting
- int getTryCount() {
- return tryCount;
- }
-
- @Override
- public long getBatchSize() {
- return batchSize;
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSSequenceFile.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSSequenceFile.java
deleted file mode 100644
index c5430ba65..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSSequenceFile.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HDFSSequenceFile extends AbstractHDFSWriter {
-
- private static final Logger logger =
- LoggerFactory.getLogger(HDFSSequenceFile.class);
- private SequenceFile.Writer writer;
- private String writeFormat;
- private Context serializerContext;
- private SequenceFileSerializer serializer;
- private boolean useRawLocalFileSystem;
- private FSDataOutputStream outStream = null;
-
- public HDFSSequenceFile() {
- writer = null;
- }
-
- @Override
- public void configure(Context context) {
- super.configure(context);
-
- // use binary writable serialize by default
- writeFormat = context.getString("hdfs.writeFormat",
- SequenceFileSerializerType.Writable.name());
- useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
- false);
- serializerContext = new Context(
- context.getSubProperties(SequenceFileSerializerFactory.CTX_PREFIX));
- serializer = SequenceFileSerializerFactory
- .getSerializer(writeFormat, serializerContext);
- logger.info("writeFormat = " + writeFormat + ", UseRawLocalFileSystem = "
- + useRawLocalFileSystem);
- }
-
- @Override
- public void open(String filePath) throws IOException {
- open(filePath, null, CompressionType.NONE);
- }
-
- @Override
- public void open(String filePath, CompressionCodec codeC,
- CompressionType compType) throws IOException {
- Configuration conf = new Configuration();
- Path dstPath = new Path(filePath);
- FileSystem hdfs = dstPath.getFileSystem(conf);
- open(dstPath, codeC, compType, conf, hdfs);
- }
-
- protected void open(Path dstPath, CompressionCodec codeC,
- CompressionType compType, Configuration conf, FileSystem hdfs)
- throws IOException {
- if (useRawLocalFileSystem) {
- if (hdfs instanceof LocalFileSystem) {
- hdfs = ((LocalFileSystem)hdfs).getRaw();
- } else {
- logger.warn("useRawLocalFileSystem is set to true but file system " +
- "is not of type LocalFileSystem: " + hdfs.getClass().getName());
- }
- }
- if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) {
- outStream = hdfs.append(dstPath);
- } else {
- outStream = hdfs.create(dstPath);
- }
- writer = SequenceFile.createWriter(conf, outStream,
- serializer.getKeyClass(), serializer.getValueClass(), compType, codeC);
-
- registerCurrentStream(outStream, hdfs, dstPath);
- }
-
- @Override
- public void append(Event e) throws IOException {
- for (SequenceFileSerializer.Record record : serializer.serialize(e)) {
- writer.append(record.getKey(), record.getValue());
- }
- }
-
- @Override
- public void sync() throws IOException {
- writer.sync();
- hflushOrSync(outStream);
- }
-
- @Override
- public void close() throws IOException {
- writer.close();
- outStream.close();
- unregisterCurrentStream();
- }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextSerializer.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextSerializer.java
deleted file mode 100644
index 32fd20662..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextSerializer.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.util.Collections;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.LongWritable;
-
-public class HDFSTextSerializer implements SequenceFileSerializer {
-
- private Text makeText(Event e) {
- Text textObject = new Text();
- textObject.set(e.getBody(), 0, e.getBody().length);
- return textObject;
- }
-
- @Override
- public Class<LongWritable> getKeyClass() {
- return LongWritable.class;
- }
-
- @Override
- public Class<Text> getValueClass() {
- return Text.class;
- }
-
- @Override
- public Iterable<Record> serialize(Event e) {
- Object key = getKey(e);
- Object value = getValue(e);
- return Collections.singletonList(new Record(key, value));
- }
-
- private Object getKey(Event e) {
- // Write the data to HDFS
- String timestamp = e.getHeaders().get("timestamp");
- long eventStamp;
-
- if (timestamp == null) {
- eventStamp = System.currentTimeMillis();
- } else {
- eventStamp = Long.valueOf(timestamp);
- }
- return new LongWritable(eventStamp);
- }
-
- private Object getValue(Event e) {
- return makeText(e);
- }
-
- public static class Builder implements SequenceFileSerializer.Builder {
-
- @Override
- public SequenceFileSerializer build(Context context) {
- return new HDFSTextSerializer();
- }
-
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableSerializer.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableSerializer.java
deleted file mode 100644
index b25a6ea50..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableSerializer.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.LongWritable;
-
-import java.util.Collections;
-
-public class HDFSWritableSerializer implements SequenceFileSerializer {
-
- private BytesWritable makeByteWritable(Event e) {
- BytesWritable bytesObject = new BytesWritable();
- bytesObject.set(e.getBody(), 0, e.getBody().length);
- return bytesObject;
- }
-
- @Override
- public Class<LongWritable> getKeyClass() {
- return LongWritable.class;
- }
-
- @Override
- public Class<BytesWritable> getValueClass() {
- return BytesWritable.class;
- }
-
- @Override
- public Iterable<Record> serialize(Event e) {
- Object key = getKey(e);
- Object value = getValue(e);
- return Collections.singletonList(new Record(key, value));
- }
-
- private Object getKey(Event e) {
- String timestamp = e.getHeaders().get("timestamp");
- long eventStamp;
-
- if (timestamp == null) {
- eventStamp = System.currentTimeMillis();
- } else {
- eventStamp = Long.valueOf(timestamp);
- }
- return new LongWritable(eventStamp);
- }
-
- private Object getValue(Event e) {
- return makeByteWritable(e);
- }
-
- public static class Builder implements SequenceFileSerializer.Builder {
-
- @Override
- public SequenceFileSerializer build(Context context) {
- return new HDFSWritableSerializer();
- }
-
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriter.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriter.java
deleted file mode 100644
index 44a984ac4..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-
-import org.apache.flume.Event;
-import org.apache.flume.annotations.InterfaceAudience;
-import org.apache.flume.annotations.InterfaceStability;
-import org.apache.flume.conf.Configurable;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface HDFSWriter extends Configurable {
-
- public void open(String filePath) throws IOException;
-
- public void open(String filePath, CompressionCodec codec,
- CompressionType cType) throws IOException;
-
- public void append(Event e) throws IOException;
-
- public void sync() throws IOException;
-
- public void close() throws IOException;
-
- public boolean isUnderReplicated();
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriterFactory.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriterFactory.java
deleted file mode 100644
index a90d53643..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriterFactory.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-
-public class HDFSWriterFactory {
- static final String SequenceFileType = "SequenceFile";
- static final String DataStreamType = "DataStream";
- static final String CompStreamType = "CompressedStream";
-
- public HDFSWriterFactory() {
-
- }
-
- public HDFSWriter getWriter(String fileType) throws IOException {
- if (fileType.equalsIgnoreCase(SequenceFileType)) {
- return new HDFSSequenceFile();
- } else if (fileType.equalsIgnoreCase(DataStreamType)) {
- return new HDFSDataStream();
- } else if (fileType.equalsIgnoreCase(CompStreamType)) {
- return new HDFSCompressedDataStream();
- } else {
- throw new IOException("File type " + fileType + " not supported");
- }
- }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/KerberosUser.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/KerberosUser.java
deleted file mode 100644
index 43297e25b..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/KerberosUser.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.flume.sink.hdfs;
-
-/**
- * Simple Pair class used to define a unique (principal, keyTab) combination.
- */
-public class KerberosUser {
-
- private final String principal;
- private final String keyTab;
-
- public KerberosUser(String principal, String keyTab) {
- this.principal = principal;
- this.keyTab = keyTab;
- }
-
- public String getPrincipal() {
- return principal;
- }
-
- public String getKeyTab() {
- return keyTab;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- final KerberosUser other = (KerberosUser) obj;
- if ((this.principal == null) ?
- (other.principal != null) :
- !this.principal.equals(other.principal)) {
- return false;
- }
- if ((this.keyTab == null) ? (other.keyTab != null) : !this.keyTab.equals(other.keyTab)) {
- return false;
- }
- return true;
- }
-
- @Override
- public int hashCode() {
- int hash = 7;
- hash = 41 * hash + (this.principal != null ? this.principal.hashCode() : 0);
- hash = 41 * hash + (this.keyTab != null ? this.keyTab.hashCode() : 0);
- return hash;
- }
-
- @Override
- public String toString() {
- return "{ principal: " + principal + ", keytab: " + keyTab + " }";
- }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializer.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializer.java
deleted file mode 100644
index ec2b7601d..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializer.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-
-public interface SequenceFileSerializer {
-
- Class<?> getKeyClass();
-
- Class<?> getValueClass();
-
- /**
- * Format the given event into zero, one or more SequenceFile records
- *
- * @param e
- * event
- * @return a list of records corresponding to the given event
- */
- Iterable<Record> serialize(Event e);
-
- /**
- * Knows how to construct this output formatter.<br/>
- * <b>Note: Implementations MUST provide a public a no-arg constructor.</b>
- */
- public interface Builder {
- public SequenceFileSerializer build(Context context);
- }
-
- /**
- * A key-value pair making up a record in an HDFS SequenceFile
- */
- public static class Record {
- private final Object key;
- private final Object value;
-
- public Record(Object key, Object value) {
- this.key = key;
- this.value = value;
- }
-
- public Object getKey() {
- return key;
- }
-
- public Object getValue() {
- return value;
- }
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerFactory.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerFactory.java
deleted file mode 100644
index 56788364f..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerFactory.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import com.google.common.base.Preconditions;
-import org.apache.flume.Context;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class SequenceFileSerializerFactory {
-
- private static final Logger logger =
- LoggerFactory.getLogger(SequenceFileSerializerFactory.class);
-
- /**
- * {@link Context} prefix
- */
- static final String CTX_PREFIX = "writeFormat.";
-
- @SuppressWarnings("unchecked")
- static SequenceFileSerializer getSerializer(String formatType,
- Context context) {
-
- Preconditions.checkNotNull(formatType,
- "serialize type must not be null");
-
- // try to find builder class in enum of known formatters
- SequenceFileSerializerType type;
- try {
- type = SequenceFileSerializerType.valueOf(formatType);
- } catch (IllegalArgumentException e) {
- logger.debug("Not in enum, loading builder class: {}", formatType);
- type = SequenceFileSerializerType.Other;
- }
- Class<? extends SequenceFileSerializer.Builder> builderClass =
- type.getBuilderClass();
-
- // handle the case where they have specified their own builder in the config
- if (builderClass == null) {
- try {
- Class c = Class.forName(formatType);
- if (c != null && SequenceFileSerializer.Builder.class.isAssignableFrom(c)) {
- builderClass = (Class<? extends SequenceFileSerializer.Builder>) c;
- } else {
- logger.error("Unable to instantiate Builder from {}", formatType);
- return null;
- }
- } catch (ClassNotFoundException ex) {
- logger.error("Class not found: " + formatType, ex);
- return null;
- } catch (ClassCastException ex) {
- logger.error("Class does not extend " +
- SequenceFileSerializer.Builder.class.getCanonicalName() + ": " +
- formatType, ex);
- return null;
- }
- }
-
- // build the builder
- SequenceFileSerializer.Builder builder;
- try {
- builder = builderClass.newInstance();
- } catch (InstantiationException ex) {
- logger.error("Cannot instantiate builder: " + formatType, ex);
- return null;
- } catch (IllegalAccessException ex) {
- logger.error("Cannot instantiate builder: " + formatType, ex);
- return null;
- }
-
- return builder.build(context);
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerType.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerType.java
deleted file mode 100644
index 2ad7689f2..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerType.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-public enum SequenceFileSerializerType {
- Writable(HDFSWritableSerializer.Builder.class),
- Text(HDFSTextSerializer.Builder.class),
- Other(null);
-
- private final Class<? extends SequenceFileSerializer.Builder> builderClass;
-
- SequenceFileSerializerType(Class<? extends SequenceFileSerializer.Builder> builderClass) {
- this.builderClass = builderClass;
- }
-
- public Class<? extends SequenceFileSerializer.Builder> getBuilderClass() {
- return builderClass;
- }
-
-}
-
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSBadDataStream.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSBadDataStream.java
deleted file mode 100644
index d325233ac..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSBadDataStream.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-import org.apache.flume.Event;
-
-public class HDFSBadDataStream extends HDFSDataStream {
- public class HDFSBadSeqWriter extends HDFSSequenceFile {
- @Override
- public void append(Event e) throws IOException {
-
- if (e.getHeaders().containsKey("fault")) {
- throw new IOException("Injected fault");
- } else if (e.getHeaders().containsKey("slow")) {
- long waitTime = Long.parseLong(e.getHeaders().get("slow"));
- try {
- Thread.sleep(waitTime);
- } catch (InterruptedException eT) {
- throw new IOException("append interrupted", eT);
- }
- }
- super.append(e);
- }
-
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestSeqWriter.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestSeqWriter.java
deleted file mode 100644
index f1dadf1f2..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestSeqWriter.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import org.apache.flume.Event;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-
-import java.io.IOException;
-
-public class HDFSTestSeqWriter extends HDFSSequenceFile {
- protected volatile boolean closed;
- protected volatile boolean opened;
-
- private int openCount = 0;
-
- HDFSTestSeqWriter(int openCount) {
- this.openCount = openCount;
- }
-
- @Override
- public void open(String filePath, CompressionCodec codeC, CompressionType compType)
- throws IOException {
- super.open(filePath, codeC, compType);
- if (closed) {
- opened = true;
- }
- }
-
- @Override
- public void append(Event e) throws IOException {
-
- if (e.getHeaders().containsKey("fault")) {
- throw new IOException("Injected fault");
- } else if (e.getHeaders().containsKey("fault-once")) {
- e.getHeaders().remove("fault-once");
- throw new IOException("Injected fault");
- } else if (e.getHeaders().containsKey("fault-until-reopen")) {
- // opening first time.
- if (openCount == 1) {
- throw new IOException("Injected fault-until-reopen");
- }
- } else if (e.getHeaders().containsKey("slow")) {
- long waitTime = Long.parseLong(e.getHeaders().get("slow"));
- try {
- Thread.sleep(waitTime);
- } catch (InterruptedException eT) {
- throw new IOException("append interrupted", eT);
- }
- }
-
- super.append(e);
- }
-
- @Override
- public void close() throws IOException {
- closed = true;
- super.close();
- }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestWriterFactory.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestWriterFactory.java
deleted file mode 100644
index 70bd9e643..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestWriterFactory.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-public class HDFSTestWriterFactory extends HDFSWriterFactory {
- static final String TestSequenceFileType = "SequenceFile";
- static final String BadDataStreamType = "DataStream";
-
- // so we can get a handle to this one in our test.
- AtomicInteger openCount = new AtomicInteger(0);
-
- @Override
- public HDFSWriter getWriter(String fileType) throws IOException {
- if (fileType == TestSequenceFileType) {
- return new HDFSTestSeqWriter(openCount.incrementAndGet());
- } else if (fileType == BadDataStreamType) {
- return new HDFSBadDataStream();
- } else {
- throw new IOException("File type " + fileType + " not supported");
- }
- }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockDataStream.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockDataStream.java
deleted file mode 100644
index a85a99f1c..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockDataStream.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-
-class MockDataStream extends HDFSDataStream {
- private final FileSystem fs;
-
- MockDataStream(FileSystem fs) {
- this.fs = fs;
- }
-
- @Override
- protected FileSystem getDfs(Configuration conf, Path dstPath) throws IOException {
- return fs;
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFileSystem.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFileSystem.java
deleted file mode 100644
index a079b8316..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFileSystem.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-import java.net.URI;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.util.Progressable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MockFileSystem extends FileSystem {
-
- private static final Logger logger =
- LoggerFactory.getLogger(MockFileSystem.class);
-
- FileSystem fs;
- int numberOfRetriesRequired;
- MockFsDataOutputStream latestOutputStream;
- int currentRenameAttempts;
- boolean closeSucceed = true;
-
- public MockFileSystem(FileSystem fs, int numberOfRetriesRequired) {
- this.fs = fs;
- this.numberOfRetriesRequired = numberOfRetriesRequired;
- }
-
- public MockFileSystem(FileSystem fs,
- int numberOfRetriesRequired, boolean closeSucceed) {
- this.fs = fs;
- this.numberOfRetriesRequired = numberOfRetriesRequired;
- this.closeSucceed = closeSucceed;
- }
-
- @Override
- public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2)
- throws IOException {
-
- latestOutputStream = new MockFsDataOutputStream(
- fs.append(arg0, arg1, arg2), closeSucceed);
-
- return latestOutputStream;
- }
-
- @Override
- public FSDataOutputStream create(Path arg0) throws IOException {
- latestOutputStream = new MockFsDataOutputStream(fs.create(arg0), closeSucceed);
- return latestOutputStream;
- }
-
- @Override
- public FSDataOutputStream create(Path arg0, FsPermission arg1, boolean arg2, int arg3,
- short arg4, long arg5, Progressable arg6)
- throws IOException {
- throw new IOException("Not a real file system");
- }
-
- @Override
- @Deprecated
- public boolean delete(Path arg0) throws IOException {
- return fs.delete(arg0);
- }
-
- @Override
- public boolean delete(Path arg0, boolean arg1) throws IOException {
- return fs.delete(arg0, arg1);
- }
-
- @Override
- public FileStatus getFileStatus(Path arg0) throws IOException {
- return fs.getFileStatus(arg0);
- }
-
- @Override
- public URI getUri() {
- return fs.getUri();
- }
-
- @Override
- public Path getWorkingDirectory() {
- return fs.getWorkingDirectory();
- }
-
- @Override
- public FileStatus[] listStatus(Path arg0) throws IOException {
- return fs.listStatus(arg0);
- }
-
- @Override
- public boolean mkdirs(Path arg0, FsPermission arg1) throws IOException {
- // TODO Auto-generated method stub
- return fs.mkdirs(arg0, arg1);
- }
-
- @Override
- public FSDataInputStream open(Path arg0, int arg1) throws IOException {
- return fs.open(arg0, arg1);
- }
-
- @Override
- public boolean rename(Path arg0, Path arg1) throws IOException {
- currentRenameAttempts++;
- logger.info("Attempting to Rename: '" + currentRenameAttempts + "' of '" +
- numberOfRetriesRequired + "'");
- if (currentRenameAttempts >= numberOfRetriesRequired || numberOfRetriesRequired == 0) {
- logger.info("Renaming file");
- return fs.rename(arg0, arg1);
- } else {
- throw new IOException("MockIOException");
- }
- }
-
- @Override
- public void setWorkingDirectory(Path arg0) {
- fs.setWorkingDirectory(arg0);
- }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFsDataOutputStream.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFsDataOutputStream.java
deleted file mode 100644
index f5d579c70..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFsDataOutputStream.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
-+ * Licensed to the Apache Software Foundation (ASF) under one
-+ * or more contributor license agreements. See the NOTICE file
-+ * distributed with this work for additional information
-+ * regarding copyright ownership. The ASF licenses this file
-+ * to you under the Apache License, Version 2.0 (the
-+ * "License"); you may not use this file except in compliance
-+ * with the License. You may obtain a copy of the License at
-+ *
-+ * http://www.apache.org/licenses/LICENSE-2.0
-+ *
-+ * Unless required by applicable law or agreed to in writing, software
-+ * distributed under the License is distributed on an "AS IS" BASIS,
-+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+ * See the License for the specific language governing permissions and
-+ * limitations under the License.
-+ */
-package org.apache.flume.sink.hdfs;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-public class MockFsDataOutputStream extends FSDataOutputStream {
-
- private static final Logger logger =
- LoggerFactory.getLogger(MockFsDataOutputStream.class);
-
- boolean closeSucceed;
-
- public MockFsDataOutputStream(FSDataOutputStream wrapMe, boolean closeSucceed)
- throws IOException {
- super(wrapMe.getWrappedStream(), null);
- this.closeSucceed = closeSucceed;
- }
-
- @Override
- public void close() throws IOException {
- logger.info("Close Succeeded - " + closeSucceed);
- if (closeSucceed) {
- logger.info("closing file");
- super.close();
- } else {
- throw new IOException("MockIOException");
- }
- }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockHDFSWriter.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockHDFSWriter.java
deleted file mode 100644
index c57496bb1..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockHDFSWriter.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MockHDFSWriter implements HDFSWriter {
-
- private int filesOpened = 0;
- private int filesClosed = 0;
- private int bytesWritten = 0;
- private int eventsWritten = 0;
- private String filePath = null;
- private static final Logger logger = LoggerFactory.getLogger(MockHDFSWriter.class);
- private int numberOfRetriesRequired;
- public volatile AtomicInteger currentCloseAttempts = new AtomicInteger(0);
-
- public MockHDFSWriter(int numberOfRetriesRequired) {
- this.numberOfRetriesRequired = numberOfRetriesRequired;
- }
-
- public MockHDFSWriter() {
- this.numberOfRetriesRequired = 0;
- }
-
- public int getFilesOpened() {
- return filesOpened;
- }
-
- public int getFilesClosed() {
- return filesClosed;
- }
-
- public int getBytesWritten() {
- return bytesWritten;
- }
-
- public int getEventsWritten() {
- return eventsWritten;
- }
-
- public String getOpenedFilePath() {
- return filePath;
- }
-
- public void configure(Context context) {
- // no-op
- }
-
- public void open(String filePath) throws IOException {
- this.filePath = filePath;
- filesOpened++;
- }
-
- public void open(String filePath, CompressionCodec codec, CompressionType cType)
- throws IOException {
- this.filePath = filePath;
- filesOpened++;
- }
-
- public void append(Event e) throws IOException {
- eventsWritten++;
- bytesWritten += e.getBody().length;
- }
-
- public void sync() throws IOException {
- // does nothing
- }
-
- public void close() throws IOException {
- filesClosed++;
- int curr = currentCloseAttempts.incrementAndGet();
- logger.info("Attempting to close: '" + currentCloseAttempts + "' of '" +
- numberOfRetriesRequired + "'");
- if (curr >= numberOfRetriesRequired || numberOfRetriesRequired == 0) {
- logger.info("closing file");
- } else {
- throw new IOException("MockIOException");
- }
- }
-
- @Override
- public boolean isUnderReplicated() {
- return false;
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomSerializer.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomSerializer.java
deleted file mode 100644
index 72164fdcf..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomSerializer.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.LongWritable;
-
-import java.util.Arrays;
-
-public class MyCustomSerializer implements SequenceFileSerializer {
-
- @Override
- public Class<LongWritable> getKeyClass() {
- return LongWritable.class;
- }
-
- @Override
- public Class<BytesWritable> getValueClass() {
- return BytesWritable.class;
- }
-
- @Override
- public Iterable<Record> serialize(Event e) {
- return Arrays.asList(
- new Record(new LongWritable(1234L), new BytesWritable(new byte[10])),
- new Record(new LongWritable(4567L), new BytesWritable(new byte[20]))
- );
- }
-
- public static class Builder implements SequenceFileSerializer.Builder {
-
- @Override
- public SequenceFileSerializer build(Context context) {
- return new MyCustomSerializer();
- }
-
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestAvroEventSerializer.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestAvroEventSerializer.java
deleted file mode 100644
index 6b38da2a0..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestAvroEventSerializer.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Arrays;
-import org.apache.avro.Schema;
-import org.apache.avro.file.DataFileReader;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.generic.GenericRecordBuilder;
-import org.apache.avro.io.BinaryEncoder;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.io.EncoderFactory;
-import org.apache.avro.reflect.ReflectDatumWriter;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.event.EventBuilder;
-import org.apache.flume.serialization.AvroEventSerializerConfigurationConstants;
-import org.apache.flume.serialization.EventSerializer;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.After;
-
-public class TestAvroEventSerializer {
-
- private File file;
-
- @Before
- public void setUp() throws Exception {
- file = File.createTempFile(getClass().getSimpleName(), "");
- }
-
- @After
- public void tearDown() throws Exception {
- file.delete();
- }
-
- @Test
- public void testNoCompression() throws IOException {
- createAvroFile(file, null, false, false);
- validateAvroFile(file);
- }
-
- @Test
- public void testNullCompression() throws IOException {
- createAvroFile(file, "null", false, false);
- validateAvroFile(file);
- }
-
- @Test
- public void testDeflateCompression() throws IOException {
- createAvroFile(file, "deflate", false, false);
- validateAvroFile(file);
- }
-
- @Test
- public void testSnappyCompression() throws IOException {
- createAvroFile(file, "snappy", false, false);
- validateAvroFile(file);
- }
-
- @Test
- public void testSchemaUrl() throws IOException {
- createAvroFile(file, null, true, false);
- validateAvroFile(file);
- }
-
- @Test
- public void testStaticSchemaUrl() throws IOException {
- createAvroFile(file,null,false, true);
- validateAvroFile(file);
- }
-
- @Test
- public void testBothUrls() throws IOException {
- createAvroFile(file,null,true,true);
- validateAvroFile(file);
- }
-
- public void createAvroFile(File file, String codec, boolean useSchemaUrl,
- boolean useStaticSchemaUrl) throws IOException {
- // serialize a few events using the reflection-based avro serializer
- OutputStream out = new FileOutputStream(file);
-
- Context ctx = new Context();
- if (codec != null) {
- ctx.put("compressionCodec", codec);
- }
-
- Schema schema = Schema.createRecord("myrecord", null, null, false);
- schema.setFields(Arrays.asList(new Schema.Field[]{
- new Schema.Field("message", Schema.create(Schema.Type.STRING), null, null)
- }));
- GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
- File schemaFile = null;
- if (useSchemaUrl || useStaticSchemaUrl) {
- schemaFile = File.createTempFile(getClass().getSimpleName(), ".avsc");
- Files.write(schema.toString(), schemaFile, Charsets.UTF_8);
- }
-
- if (useStaticSchemaUrl) {
- ctx.put(AvroEventSerializerConfigurationConstants.STATIC_SCHEMA_URL,
- schemaFile.toURI().toURL().toExternalForm());
- }
-
- EventSerializer.Builder builder = new AvroEventSerializer.Builder();
- EventSerializer serializer = builder.build(ctx, out);
-
- serializer.afterCreate();
- for (int i = 0; i < 3; i++) {
- GenericRecord record = recordBuilder.set("message", "Hello " + i).build();
- Event event = EventBuilder.withBody(serializeAvro(record, schema));
- if (schemaFile == null && !useSchemaUrl) {
- event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_LITERAL_HEADER,
- schema.toString());
- } else if (useSchemaUrl) {
- event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_URL_HEADER,
- schemaFile.toURI().toURL().toExternalForm());
- }
- serializer.write(event);
- }
- serializer.flush();
- serializer.beforeClose();
- out.flush();
- out.close();
- if (schemaFile != null ) {
- schemaFile.delete();
- }
-
- }
-
- private byte[] serializeAvro(Object datum, Schema schema) throws IOException {
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- ReflectDatumWriter<Object> writer = new ReflectDatumWriter<Object>(schema);
- BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
- out.reset();
- writer.write(datum, encoder);
- encoder.flush();
- return out.toByteArray();
- }
-
- public void validateAvroFile(File file) throws IOException {
- // read the events back using GenericRecord
- DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
- DataFileReader<GenericRecord> fileReader =
- new DataFileReader<GenericRecord>(file, reader);
- GenericRecord record = new GenericData.Record(fileReader.getSchema());
- int numEvents = 0;
- while (fileReader.hasNext()) {
- fileReader.next(record);
- String bodyStr = record.get("message").toString();
- System.out.println(bodyStr);
- numEvents++;
- }
- fileReader.close();
- Assert.assertEquals("Should have found a total of 3 events", 3, numEvents);
- }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestBucketWriter.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestBucketWriter.java
deleted file mode 100644
index 64a89e171..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestBucketWriter.java
+++ /dev/null
@@ -1,719 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import com.google.common.base.Charsets;
-import org.apache.flume.Clock;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.SystemClock;
-import org.apache.flume.auth.FlumeAuthenticationUtil;
-import org.apache.flume.auth.PrivilegedExecutor;
-import org.apache.flume.event.EventBuilder;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.flume.sink.hdfs.HDFSEventSink.WriterCallback;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Calendar;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class TestBucketWriter {
-
- private static Logger logger = LoggerFactory.getLogger(TestBucketWriter.class);
- private Context ctx = new Context();
-
- private static ScheduledExecutorService timedRollerPool;
- private static PrivilegedExecutor proxy;
-
- @BeforeClass
- public static void setup() {
- timedRollerPool = Executors.newSingleThreadScheduledExecutor();
- proxy = FlumeAuthenticationUtil.getAuthenticator(null, null).proxyAs(null);
- }
-
- @AfterClass
- public static void teardown() throws InterruptedException {
- timedRollerPool.shutdown();
- timedRollerPool.awaitTermination(2, TimeUnit.SECONDS);
- timedRollerPool.shutdownNow();
- }
-
- @Test
- public void testEventCountingRoller() throws IOException, InterruptedException {
- int maxEvents = 100;
- MockHDFSWriter hdfsWriter = new MockHDFSWriter();
- BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
- .setRollCount(maxEvents).build();
-
- Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
- for (int i = 0; i < 1000; i++) {
- bucketWriter.append(e);
- }
-
- logger.info("Number of events written: {}", hdfsWriter.getEventsWritten());
- logger.info("Number of bytes written: {}", hdfsWriter.getBytesWritten());
- logger.info("Number of files opened: {}", hdfsWriter.getFilesOpened());
-
- Assert.assertEquals("events written", 1000, hdfsWriter.getEventsWritten());
- Assert.assertEquals("bytes written", 3000, hdfsWriter.getBytesWritten());
- Assert.assertEquals("files opened", 10, hdfsWriter.getFilesOpened());
- }
-
- @Test
- public void testSizeRoller() throws IOException, InterruptedException {
- int maxBytes = 300;
- MockHDFSWriter hdfsWriter = new MockHDFSWriter();
- BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
- .setRollSize(maxBytes).build();
-
- Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
- for (int i = 0; i < 1000; i++) {
- bucketWriter.append(e);
- }
-
- logger.info("Number of events written: {}", hdfsWriter.getEventsWritten());
- logger.info("Number of bytes written: {}", hdfsWriter.getBytesWritten());
- logger.info("Number of files opened: {}", hdfsWriter.getFilesOpened());
-
- Assert.assertEquals("events written", 1000, hdfsWriter.getEventsWritten());
- Assert.assertEquals("bytes written", 3000, hdfsWriter.getBytesWritten());
- Assert.assertEquals("files opened", 10, hdfsWriter.getFilesOpened());
- }
-
- @Test
- public void testIntervalRoller() throws IOException, InterruptedException {
- final int ROLL_INTERVAL = 1; // seconds
- final int NUM_EVENTS = 10;
- final AtomicBoolean calledBack = new AtomicBoolean(false);
-
- MockHDFSWriter hdfsWriter = new MockHDFSWriter();
- BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
- .setRollInterval(ROLL_INTERVAL)
- .setOnCloseCallback(new HDFSEventSink.WriterCallback() {
- @Override
- public void run(String filePath) {
- calledBack.set(true);
- }
- }).build();
-
- Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
- long startNanos = System.nanoTime();
- for (int i = 0; i < NUM_EVENTS - 1; i++) {
- bucketWriter.append(e);
- }
-
- // sleep to force a roll... wait 2x interval just to be sure
- Thread.sleep(2 * ROLL_INTERVAL * 1000L);
-
- Assert.assertTrue(bucketWriter.closed.get());
- Assert.assertTrue(calledBack.get());
-
- bucketWriter = new BucketWriterBuilder(hdfsWriter)
- .setRollInterval(ROLL_INTERVAL).build();
- // write one more event (to reopen a new file so we will roll again later)
- bucketWriter.append(e);
-
- long elapsedMillis = TimeUnit.MILLISECONDS.convert(
- System.nanoTime() - startNanos, TimeUnit.NANOSECONDS);
- long elapsedSeconds = elapsedMillis / 1000L;
-
- logger.info("Time elapsed: {} milliseconds", elapsedMillis);
- logger.info("Number of events written: {}", hdfsWriter.getEventsWritten());
- logger.info("Number of bytes written: {}", hdfsWriter.getBytesWritten());
- logger.info("Number of files opened: {}", hdfsWriter.getFilesOpened());
- logger.info("Number of files closed: {}", hdfsWriter.getFilesClosed());
-
- Assert.assertEquals("events written", NUM_EVENTS,
- hdfsWriter.getEventsWritten());
- Assert.assertEquals("bytes written", e.getBody().length * NUM_EVENTS,
- hdfsWriter.getBytesWritten());
- Assert.assertEquals("files opened", 2, hdfsWriter.getFilesOpened());
-
- // before auto-roll
- Assert.assertEquals("files closed", 1, hdfsWriter.getFilesClosed());
-
- logger.info("Waiting for roll...");
- Thread.sleep(2 * ROLL_INTERVAL * 1000L);
-
- logger.info("Number of files closed: {}", hdfsWriter.getFilesClosed());
- Assert.assertEquals("files closed", 2, hdfsWriter.getFilesClosed());
- }
-
- @Test
- public void testIntervalRollerBug() throws IOException, InterruptedException {
- final int ROLL_INTERVAL = 1; // seconds
- final int NUM_EVENTS = 10;
-
- HDFSWriter hdfsWriter = new HDFSWriter() {
- private volatile boolean open = false;
-
- public void configure(Context context) {
- }
-
- public void sync() throws IOException {
- if (!open) {
- throw new IOException("closed");
- }
- }
-
- public void open(String filePath, CompressionCodec codec, CompressionType cType)
- throws IOException {
- open = true;
- }
-
- public void open(String filePath) throws IOException {
- open = true;
- }
-
- public void close() throws IOException {
- open = false;
- }
-
- @Override
- public boolean isUnderReplicated() {
- return false;
- }
-
- public void append(Event e) throws IOException {
- // we just re-open in append if closed
- open = true;
- }
- };
-
-
- File tmpFile = File.createTempFile("flume", "test");
- tmpFile.deleteOnExit();
- String path = tmpFile.getParent();
- String name = tmpFile.getName();
-
- BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
- .setRollInterval(ROLL_INTERVAL)
- .setFilePath(path)
- .setFileName(name)
- .build();
-
- Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
- for (int i = 0; i < NUM_EVENTS - 1; i++) {
- bucketWriter.append(e);
- }
-
- // sleep to force a roll... wait 2x interval just to be sure
- Thread.sleep(2 * ROLL_INTERVAL * 1000L);
-
- bucketWriter.flush(); // throws closed exception
- }
-
- @Test
- public void testFileSuffixNotGiven() throws IOException, InterruptedException {
- final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
- final String suffix = null;
-
- // Need to override system time use for test so we know what to expect
- final long testTime = System.currentTimeMillis();
- Clock testClock = new Clock() {
- public long currentTimeMillis() {
- return testTime;
- }
- };
-
- MockHDFSWriter hdfsWriter = new MockHDFSWriter();
- BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
- .setRollInterval(ROLL_INTERVAL)
- .setFileSuffix(suffix)
- .setClock(testClock)
- .build();
-
- Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
- bucketWriter.append(e);
-
- Assert.assertTrue("Incorrect suffix", hdfsWriter.getOpenedFilePath().endsWith(
- Long.toString(testTime + 1) + ".tmp"));
- }
-
- @Test
- public void testFileSuffixGiven() throws IOException, InterruptedException {
- final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
- final String suffix = ".avro";
-
- // Need to override system time use for test so we know what to expect
-
- final long testTime = System.currentTimeMillis();
-
- Clock testClock = new Clock() {
- public long currentTimeMillis() {
- return testTime;
- }
- };
-
- MockHDFSWriter hdfsWriter = new MockHDFSWriter();
- BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
- .setRollInterval(ROLL_INTERVAL)
- .setFileSuffix(suffix)
- .setClock(testClock)
- .build();
-
-
- Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
- bucketWriter.append(e);
-
- Assert.assertTrue("Incorrect suffix",hdfsWriter.getOpenedFilePath().endsWith(
- Long.toString(testTime + 1) + suffix + ".tmp"));
- }
-
- @Test
- public void testFileSuffixCompressed()
- throws IOException, InterruptedException {
- final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
- final String suffix = ".foo";
-
- MockHDFSWriter hdfsWriter = new MockHDFSWriter();
-
- // Need to override system time use for test so we know what to expect
- final long testTime = System.currentTimeMillis();
-
- Clock testClock = new Clock() {
- public long currentTimeMillis() {
- return testTime;
- }
- };
-
- BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
- .setRollInterval(ROLL_INTERVAL)
- .setFileSuffix(suffix)
- .setCodeC(HDFSEventSink.getCodec("gzip"))
- .setCompType(SequenceFile.CompressionType.BLOCK)
- .setClock(testClock)
- .build();
-
-
-
-
- Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
- bucketWriter.append(e);
-
- Assert.assertTrue("Incorrect suffix", hdfsWriter.getOpenedFilePath().endsWith(
- Long.toString(testTime + 1) + suffix + ".tmp"));
- }
-
- @Test
- public void testInUsePrefix() throws IOException, InterruptedException {
- final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
- final String PREFIX = "BRNO_IS_CITY_IN_CZECH_REPUBLIC";
-
- MockHDFSWriter hdfsWriter = new MockHDFSWriter();
- HDFSTextSerializer formatter = new HDFSTextSerializer();
- BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
- .setRollInterval(ROLL_INTERVAL)
- .setInUsePrefix(PREFIX)
- .build();
-
- Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
- bucketWriter.append(e);
-
- Assert.assertTrue("Incorrect in use prefix", hdfsWriter.getOpenedFilePath().contains(PREFIX));
- }
-
- @Test
- public void testInUseSuffix() throws IOException, InterruptedException {
- final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
- final String SUFFIX = "WELCOME_TO_THE_HELLMOUNTH";
-
- MockHDFSWriter hdfsWriter = new MockHDFSWriter();
- HDFSTextSerializer serializer = new HDFSTextSerializer();
- BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
- .setRollInterval(ROLL_INTERVAL)
- .setInUseSuffix(SUFFIX)
- .build();
-
- Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
- bucketWriter.append(e);
-
- Assert.assertTrue("Incorrect in use suffix", hdfsWriter.getOpenedFilePath().contains(SUFFIX));
- }
-
- @Test
- public void testCallbackOnClose() throws IOException, InterruptedException {
- final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
- final String SUFFIX = "WELCOME_TO_THE_EREBOR";
- final AtomicBoolean callbackCalled = new AtomicBoolean(false);
-
- BucketWriter bucketWriter = new BucketWriterBuilder()
- .setRollInterval(ROLL_INTERVAL)
- .setInUseSuffix(SUFFIX)
- .setOnCloseCallback(new HDFSEventSink.WriterCallback() {
- @Override
- public void run(String filePath) {
- callbackCalled.set(true);
- }
- })
- .setOnCloseCallbackPath("blah")
- .build();
-
- Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
- bucketWriter.append(e);
- bucketWriter.close(true);
-
- Assert.assertTrue(callbackCalled.get());
- }
-
- @Test
- public void testSequenceFileRenameRetries() throws Exception {
- sequenceFileRenameRetryCoreTest(1, true);
- sequenceFileRenameRetryCoreTest(5, true);
- sequenceFileRenameRetryCoreTest(2, true);
-
- sequenceFileRenameRetryCoreTest(1, false);
- sequenceFileRenameRetryCoreTest(5, false);
- sequenceFileRenameRetryCoreTest(2, false);
- }
-
- @Test
- public void testSequenceFileCloseRetries() throws Exception {
- sequenceFileCloseRetryCoreTest(5);
- sequenceFileCloseRetryCoreTest(1);
- }
-
- public void sequenceFileRenameRetryCoreTest(int numberOfRetriesRequired, boolean closeSucceed)
- throws Exception {
- String hdfsPath = "file:///tmp/flume-test." +
- Calendar.getInstance().getTimeInMillis() +
- "." + Thread.currentThread().getId();
-
- Context context = new Context();
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(hdfsPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
- context.put("hdfs.path", hdfsPath);
- context.put("hdfs.closeTries", String.valueOf(numberOfRetriesRequired));
- context.put("hdfs.rollCount", "1");
- context.put("hdfs.retryInterval", "1");
- context.put("hdfs.callTimeout", Long.toString(1000));
- MockFileSystem mockFs = new MockFileSystem(fs, numberOfRetriesRequired, closeSucceed);
- MockDataStream writer = new MockDataStream(mockFs);
- BucketWriter bucketWriter = new BucketWriterBuilder(writer)
- .setRollCount(1)
- .setBatchSize(1)
- .setFilePath(hdfsPath)
- .setFileName(hdfsPath)
- .setInUsePrefix("singleBucket")
- .setCompType(null)
- .setRetryInterval(1)
- .setMaxCloseTries(numberOfRetriesRequired)
- .setWriter(writer)
- .build();
-
- bucketWriter.setFileSystem(mockFs);
- // At this point, we checked if isFileClosed is available in
- // this JVM, so lets make it check again.
- Event event = EventBuilder.withBody("test", Charsets.UTF_8);
- bucketWriter.append(event);
- // This is what triggers the close, so a 2nd append is required :/
- bucketWriter.append(event);
-
- TimeUnit.SECONDS.sleep(numberOfRetriesRequired + 2);
-
- Assert.assertTrue("Expected " + numberOfRetriesRequired + " " +
- "but got " + bucketWriter.renameTries.get(),
- bucketWriter.renameTries.get() == numberOfRetriesRequired);
- }
-
- private void sequenceFileCloseRetryCoreTest(int numberOfRetriesRequired)
- throws Exception {
- String hdfsPath = "file:///tmp/flume-test." +
- Calendar.getInstance().getTimeInMillis() +
- "." + Thread.currentThread().getId();
-
- Context context = new Context();
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(hdfsPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
- context.put("hdfs.path", hdfsPath);
- context.put("hdfs.closeTries", String.valueOf(numberOfRetriesRequired));
- context.put("hdfs.rollCount", "1");
- context.put("hdfs.retryInterval", "1");
- context.put("hdfs.callTimeout", Long.toString(1000));
- MockHDFSWriter mockHDFSWriter = new MockHDFSWriter(Integer.MAX_VALUE);
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- BucketWriter bucketWriter = new BucketWriter(
- 0, 0, 1, 1, ctx, hdfsPath, hdfsPath, "singleBucket", ".tmp", null, null,
- null, mockHDFSWriter, timedRollerPool, proxy,
- new SinkCounter("test-bucket-writer-" + System.currentTimeMillis()), 0, null, null, 30000,
- executorService, 1, numberOfRetriesRequired);
-
- Event event = EventBuilder.withBody("test", Charsets.UTF_8);
- bucketWriter.append(event);
- bucketWriter.close(false);
- TimeUnit.SECONDS.sleep(numberOfRetriesRequired + 2);
- Assert.assertEquals("ExcceutorService should be empty",
- executorService.shutdownNow().size(), 0);
- Assert.assertEquals("Expected " + numberOfRetriesRequired + " " +
- "but got " + mockHDFSWriter.currentCloseAttempts,
- mockHDFSWriter.currentCloseAttempts.get(), numberOfRetriesRequired);
- }
-
- // Test that we don't swallow IOExceptions in secure mode. We should close the bucket writer
- // and rethrow the exception. Regression test for FLUME-3049.
- @Test
- public void testRotateBucketOnIOException() throws IOException, InterruptedException {
- MockHDFSWriter hdfsWriter = Mockito.spy(new MockHDFSWriter());
- PrivilegedExecutor ugiProxy =
- FlumeAuthenticationUtil.getAuthenticator(null, null).proxyAs("alice");
-
- final int ROLL_COUNT = 1; // Cause a roll after every successful append().
- BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
- .setProxyUser(ugiProxy)
- .setRollCount(ROLL_COUNT)
- .build();
-
- Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
-
- // Write one event successfully.
- bucketWriter.append(e);
-
- // Fail the next write.
- IOException expectedIOException = new IOException("Test injected IOException");
- Mockito.doThrow(expectedIOException).when(hdfsWriter)
- .append(Mockito.any(Event.class));
-
- // The second time we try to write we should get an IOException.
- try {
- bucketWriter.append(e);
- Assert.fail("Expected IOException wasn't thrown during append");
- } catch (IOException ex) {
- Assert.assertEquals(expectedIOException, ex);
- logger.info("Caught expected IOException", ex);
- }
-
- // The third time we try to write we should get a BucketClosedException, because the
- // BucketWriter should attempt to close itself before rethrowing the IOException on the first
- // call.
- try {
- bucketWriter.append(e);
- Assert.fail("BucketWriter should be already closed, BucketClosedException expected");
- } catch (BucketClosedException ex) {
- logger.info("Caught expected BucketClosedException", ex);
- }
-
- Assert.assertEquals("events written", 1, hdfsWriter.getEventsWritten());
- Assert.assertEquals("2 files should be closed", 2, hdfsWriter.getFilesClosed());
- }
-
- private class BucketWriterBuilder {
- private long rollInterval = 0;
- private long rollSize = 0;
- private long rollCount = 0;
- private long batchSize = 0;
- private Context context = TestBucketWriter.this.ctx;
- private String filePath = "/tmp";
- private String fileName = "file";
- private String inUsePrefix = "";
- private String inUseSuffix = ".tmp";
- private String fileSuffix = null;
- private CompressionCodec codeC = null;
- private CompressionType compType = SequenceFile.CompressionType.NONE;
- private HDFSWriter writer = null;
- private ScheduledExecutorService timedRollerPool = TestBucketWriter.timedRollerPool;
- private PrivilegedExecutor proxyUser = TestBucketWriter.proxy;
- private SinkCounter sinkCounter = new SinkCounter(
- "test-bucket-writer-" + System.currentTimeMillis());
- private int idleTimeout = 0;
- private WriterCallback onCloseCallback = null;
- private String onCloseCallbackPath = null;
- private long callTimeout = 30000;
- private ExecutorService callTimeoutPool = Executors.newSingleThreadExecutor();
- private long retryInterval = 0;
- private int maxCloseTries = 0;
- private Clock clock = null;
-
- public BucketWriterBuilder() {
- }
-
- public BucketWriterBuilder(HDFSWriter writer) {
- this.writer = writer;
- }
-
- public BucketWriterBuilder setRollInterval(long rollInterval) {
- this.rollInterval = rollInterval;
- return this;
- }
-
- public BucketWriterBuilder setRollSize(long rollSize) {
- this.rollSize = rollSize;
- return this;
- }
-
- public BucketWriterBuilder setRollCount(long rollCount) {
- this.rollCount = rollCount;
- return this;
- }
-
- public BucketWriterBuilder setBatchSize(long batchSize) {
- this.batchSize = batchSize;
- return this;
- }
-
- @SuppressWarnings("unused")
- public BucketWriterBuilder setContext(Context context) {
- this.context = context;
- return this;
- }
-
- public BucketWriterBuilder setFilePath(String filePath) {
- this.filePath = filePath;
- return this;
- }
-
- public BucketWriterBuilder setFileName(String fileName) {
- this.fileName = fileName;
- return this;
- }
-
- public BucketWriterBuilder setInUsePrefix(String inUsePrefix) {
- this.inUsePrefix = inUsePrefix;
- return this;
- }
-
- public BucketWriterBuilder setInUseSuffix(String inUseSuffix) {
- this.inUseSuffix = inUseSuffix;
- return this;
- }
-
- public BucketWriterBuilder setFileSuffix(String fileSuffix) {
- this.fileSuffix = fileSuffix;
- return this;
- }
-
- public BucketWriterBuilder setCodeC(CompressionCodec codeC) {
- this.codeC = codeC;
- return this;
- }
-
- public BucketWriterBuilder setCompType(CompressionType compType) {
- this.compType = compType;
- return this;
- }
-
- @SuppressWarnings("unused")
- public BucketWriterBuilder setTimedRollerPool(
- ScheduledExecutorService timedRollerPool) {
- this.timedRollerPool = timedRollerPool;
- return this;
- }
-
- @SuppressWarnings("unused")
- public BucketWriterBuilder setProxyUser(PrivilegedExecutor proxyUser) {
- this.proxyUser = proxyUser;
- return this;
- }
-
- @SuppressWarnings("unused")
- public BucketWriterBuilder setSinkCounter(SinkCounter sinkCounter) {
- this.sinkCounter = sinkCounter;
- return this;
- }
-
- @SuppressWarnings("unused")
- public BucketWriterBuilder setIdleTimeout(int idleTimeout) {
- this.idleTimeout = idleTimeout;
- return this;
- }
-
- public BucketWriterBuilder setOnCloseCallback(
- WriterCallback onCloseCallback) {
- this.onCloseCallback = onCloseCallback;
- return this;
- }
-
- public BucketWriterBuilder setOnCloseCallbackPath(
- String onCloseCallbackPath) {
- this.onCloseCallbackPath = onCloseCallbackPath;
- return this;
- }
-
- @SuppressWarnings("unused")
- public BucketWriterBuilder setCallTimeout(long callTimeout) {
- this.callTimeout = callTimeout;
- return this;
- }
-
- @SuppressWarnings("unused")
- public BucketWriterBuilder setCallTimeoutPool(
- ExecutorService callTimeoutPool) {
- this.callTimeoutPool = callTimeoutPool;
- return this;
- }
-
- public BucketWriterBuilder setRetryInterval(long retryInterval) {
- this.retryInterval = retryInterval;
- return this;
- }
-
- public BucketWriterBuilder setMaxCloseTries(int maxCloseTries) {
- this.maxCloseTries = maxCloseTries;
- return this;
- }
-
- public BucketWriterBuilder setWriter(HDFSWriter writer) {
- this.writer = writer;
- return this;
- }
-
- public BucketWriterBuilder setClock(Clock clock) {
- this.clock = clock;
- return this;
- }
-
- public BucketWriter build() {
- if (clock == null) {
- clock = new SystemClock();
- }
- if (writer == null) {
- writer = new MockHDFSWriter();
- }
-
- return new BucketWriter(rollInterval, rollSize, rollCount, batchSize,
- context, filePath, fileName, inUsePrefix, inUseSuffix, fileSuffix,
- codeC, compType, writer, timedRollerPool, proxyUser, sinkCounter,
- idleTimeout, onCloseCallback, onCloseCallbackPath, callTimeout,
- callTimeoutPool, retryInterval, maxCloseTries, clock);
- }
- }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSCompressedDataStream.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSCompressedDataStream.java
deleted file mode 100644
index 80f199b7d..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSCompressedDataStream.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.nio.ByteBuffer;
-import java.nio.charset.CharsetDecoder;
-import java.util.List;
-import java.util.zip.GZIPInputStream;
-
-import org.apache.avro.file.DataFileStream;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.io.DatumReader;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.event.EventBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.Lists;
-
-public class TestHDFSCompressedDataStream {
-
- private static final Logger logger =
- LoggerFactory.getLogger(TestHDFSCompressedDataStream.class);
-
- private File file;
- private String fileURI;
- private CompressionCodecFactory factory;
-
- @Before
- public void init() throws Exception {
- this.file = new File("target/test/data/foo.gz");
- this.fileURI = file.getAbsoluteFile().toURI().toString();
- logger.info("File URI: {}", fileURI);
-
- Configuration conf = new Configuration();
- // local FS must be raw in order to be Syncable
- conf.set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem");
- Path path = new Path(fileURI);
- path.getFileSystem(conf); // get FS with our conf cached
-
- this.factory = new CompressionCodecFactory(conf);
- }
-
- // make sure the data makes it to disk if we sync() the data stream
- @Test
- public void testGzipDurability() throws Exception {
- Context context = new Context();
- HDFSCompressedDataStream writer = new HDFSCompressedDataStream();
- writer.configure(context);
- writer.open(fileURI, factory.getCodec(new Path(fileURI)),
- SequenceFile.CompressionType.BLOCK);
-
- String[] bodies = { "yarf!" };
- writeBodies(writer, bodies);
-
- byte[] buf = new byte[256];
- GZIPInputStream cmpIn = new GZIPInputStream(new FileInputStream(file));
- int len = cmpIn.read(buf);
- String result = new String(buf, 0, len, Charsets.UTF_8);
- result = result.trim(); // BodyTextEventSerializer adds a newline
-
- Assert.assertEquals("input and output must match", bodies[0], result);
- }
-
- @Test
- public void testGzipDurabilityWithSerializer() throws Exception {
- Context context = new Context();
- context.put("serializer", "AVRO_EVENT");
-
- HDFSCompressedDataStream writer = new HDFSCompressedDataStream();
- writer.configure(context);
-
- writer.open(fileURI, factory.getCodec(new Path(fileURI)),
- SequenceFile.CompressionType.BLOCK);
-
- String[] bodies = { "yarf!", "yarfing!" };
- writeBodies(writer, bodies);
-
- int found = 0;
- int expected = bodies.length;
- List<String> expectedBodies = Lists.newArrayList(bodies);
-
- GZIPInputStream cmpIn = new GZIPInputStream(new FileInputStream(file));
- DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
- DataFileStream<GenericRecord> avroStream =
- new DataFileStream<GenericRecord>(cmpIn, reader);
- GenericRecord record = new GenericData.Record(avroStream.getSchema());
- while (avroStream.hasNext()) {
- avroStream.next(record);
- CharsetDecoder decoder = Charsets.UTF_8.newDecoder();
- String bodyStr = decoder.decode((ByteBuffer) record.get("body"))
- .toString();
- expectedBodies.remove(bodyStr);
- found++;
- }
- avroStream.close();
- cmpIn.close();
-
- Assert.assertTrue("Found = " + found + ", Expected = " + expected
- + ", Left = " + expectedBodies.size() + " " + expectedBodies,
- expectedBodies.size() == 0);
- }
-
- private void writeBodies(HDFSCompressedDataStream writer, String... bodies)
- throws Exception {
- for (String body : bodies) {
- Event evt = EventBuilder.withBody(body, Charsets.UTF_8);
- writer.append(evt);
- }
- writer.sync();
- }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSink.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSink.java
deleted file mode 100644
index 25bd2dab6..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSink.java
+++ /dev/null
@@ -1,1687 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.ByteBuffer;
-import java.nio.charset.CharsetDecoder;
-import java.util.Arrays;
-import java.util.Calendar;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Maps;
-import org.apache.avro.file.DataFileStream;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.io.DatumReader;
-import org.apache.commons.lang.StringUtils;
-import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
-import org.apache.flume.Clock;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.Sink.Status;
-import org.apache.flume.SystemClock;
-import org.apache.flume.Transaction;
-import org.apache.flume.channel.BasicTransactionSemantics;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.conf.Configurables;
-import org.apache.flume.event.EventBuilder;
-import org.apache.flume.event.SimpleEvent;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.flume.lifecycle.LifecycleException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.mockito.internal.util.reflection.Whitebox;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.Lists;
-
-public class TestHDFSEventSink {
-
- private HDFSEventSink sink;
- private String testPath;
- private static final Logger LOG = LoggerFactory
- .getLogger(HDFSEventSink.class);
-
- static {
- System.setProperty("java.security.krb5.realm", "flume");
- System.setProperty("java.security.krb5.kdc", "blah");
- }
-
- private void dirCleanup() {
- Configuration conf = new Configuration();
- try {
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(testPath);
- if (fs.exists(dirPath)) {
- fs.delete(dirPath, true);
- }
- } catch (IOException eIO) {
- LOG.warn("IO Error in test cleanup", eIO);
- }
- }
-
- // TODO: use System.getProperty("file.separator") instead of hardcoded '/'
- @Before
- public void setUp() {
- LOG.debug("Starting...");
- /*
- * FIXME: Use a dynamic path to support concurrent test execution. Also,
- * beware of the case where this path is used for something or when the
- * Hadoop config points at file:/// rather than hdfs://. We need to find a
- * better way of testing HDFS related functionality.
- */
- testPath = "file:///tmp/flume-test."
- + Calendar.getInstance().getTimeInMillis() + "."
- + Thread.currentThread().getId();
-
- sink = new HDFSEventSink();
- sink.setName("HDFSEventSink-" + UUID.randomUUID().toString());
- dirCleanup();
- }
-
- @After
- public void tearDown() {
- if (System.getenv("hdfs_keepFiles") == null) dirCleanup();
- }
-
- @Test
- public void testTextBatchAppend() throws Exception {
- doTestTextBatchAppend(false);
- }
-
- @Test
- public void testTextBatchAppendRawFS() throws Exception {
- doTestTextBatchAppend(true);
- }
-
- public void doTestTextBatchAppend(boolean useRawLocalFileSystem)
- throws Exception {
- LOG.debug("Starting...");
-
- final long rollCount = 10;
- final long batchSize = 2;
- final String fileName = "FlumeData";
- String newPath = testPath + "/singleTextBucket";
- int totalEvents = 0;
- int i = 1, j = 1;
-
- // clear the test directory
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(newPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
-
- Context context = new Context();
-
- // context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");
- context.put("hdfs.path", newPath);
- context.put("hdfs.filePrefix", fileName);
- context.put("hdfs.rollCount", String.valueOf(rollCount));
- context.put("hdfs.rollInterval", "0");
- context.put("hdfs.rollSize", "0");
- context.put("hdfs.batchSize", String.valueOf(batchSize));
- context.put("hdfs.writeFormat", "Text");
- context.put("hdfs.useRawLocalFileSystem",
- Boolean.toString(useRawLocalFileSystem));
- context.put("hdfs.fileType", "DataStream");
-
- Configurables.configure(sink, context);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, context);
-
- sink.setChannel(channel);
- sink.start();
-
- Calendar eventDate = Calendar.getInstance();
- List<String> bodies = Lists.newArrayList();
-
- // push the event batches into channel to roll twice
- for (i = 1; i <= (rollCount * 10) / batchSize; i++) {
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (j = 1; j <= batchSize; j++) {
- Event event = new SimpleEvent();
- eventDate.clear();
- eventDate.set(2011, i, i, i, 0); // yy mm dd
- String body = "Test." + i + "." + j;
- event.setBody(body.getBytes());
- bodies.add(body);
- channel.put(event);
- totalEvents++;
- }
- txn.commit();
- txn.close();
-
- // execute sink to process the events
- sink.process();
- }
-
- sink.stop();
-
- // loop through all the files generated and check their contains
- FileStatus[] dirStat = fs.listStatus(dirPath);
- Path[] fList = FileUtil.stat2Paths(dirStat);
-
- // check that the roll happened correctly for the given data
- long expectedFiles = totalEvents / rollCount;
- if (totalEvents % rollCount > 0) expectedFiles++;
- Assert.assertEquals("num files wrong, found: " +
- Lists.newArrayList(fList), expectedFiles, fList.length);
- // check the contents of the all files
- verifyOutputTextFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
- }
-
- @Test
- public void testLifecycle() throws InterruptedException, LifecycleException {
- LOG.debug("Starting...");
- Context context = new Context();
-
- context.put("hdfs.path", testPath);
- /*
- * context.put("hdfs.rollInterval", String.class);
- * context.get("hdfs.rollSize", String.class); context.get("hdfs.rollCount",
- * String.class);
- */
- Configurables.configure(sink, context);
-
- sink.setChannel(new MemoryChannel());
-
- sink.start();
- sink.stop();
- }
-
- @Test
- public void testEmptyChannelResultsInStatusBackoff()
- throws InterruptedException, LifecycleException, EventDeliveryException {
- LOG.debug("Starting...");
- Context context = new Context();
- Channel channel = new MemoryChannel();
- context.put("hdfs.path", testPath);
- context.put("keep-alive", "0");
- Configurables.configure(sink, context);
- Configurables.configure(channel, context);
- sink.setChannel(channel);
- sink.start();
- Assert.assertEquals(Status.BACKOFF, sink.process());
- sink.stop();
- }
-
- @Test
- public void testKerbFileAccess() throws InterruptedException,
- LifecycleException, EventDeliveryException, IOException {
- LOG.debug("Starting testKerbFileAccess() ...");
- final String fileName = "FlumeData";
- final long rollCount = 5;
- final long batchSize = 2;
- String newPath = testPath + "/singleBucket";
- String kerbConfPrincipal = "user1/localhost@EXAMPLE.COM";
- String kerbKeytab = "/usr/lib/flume/nonexistkeytabfile";
-
- //turn security on
- Configuration conf = new Configuration();
- conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
- "kerberos");
- UserGroupInformation.setConfiguration(conf);
-
- Context context = new Context();
- context.put("hdfs.path", newPath);
- context.put("hdfs.filePrefix", fileName);
- context.put("hdfs.rollCount", String.valueOf(rollCount));
- context.put("hdfs.batchSize", String.valueOf(batchSize));
- context.put("hdfs.kerberosPrincipal", kerbConfPrincipal);
- context.put("hdfs.kerberosKeytab", kerbKeytab);
-
- try {
- Configurables.configure(sink, context);
- Assert.fail("no exception thrown");
- } catch (IllegalArgumentException expected) {
- Assert.assertTrue(expected.getMessage().contains(
- "Keytab is not a readable file"));
- } finally {
- //turn security off
- conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
- "simple");
- UserGroupInformation.setConfiguration(conf);
- }
- }
-
- @Test
- public void testTextAppend() throws InterruptedException, LifecycleException,
- EventDeliveryException, IOException {
-
- LOG.debug("Starting...");
- final long rollCount = 3;
- final long batchSize = 2;
- final String fileName = "FlumeData";
- String newPath = testPath + "/singleTextBucket";
- int totalEvents = 0;
- int i = 1, j = 1;
-
- // clear the test directory
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(newPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
-
- Context context = new Context();
-
- // context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");
- context.put("hdfs.path", newPath);
- context.put("hdfs.filePrefix", fileName);
- context.put("hdfs.rollCount", String.valueOf(rollCount));
- context.put("hdfs.batchSize", String.valueOf(batchSize));
- context.put("hdfs.writeFormat", "Text");
- context.put("hdfs.fileType", "DataStream");
-
- Configurables.configure(sink, context);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, context);
-
- sink.setChannel(channel);
- sink.start();
-
- Calendar eventDate = Calendar.getInstance();
- List<String> bodies = Lists.newArrayList();
-
- // push the event batches into channel
- for (i = 1; i < 4; i++) {
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (j = 1; j <= batchSize; j++) {
- Event event = new SimpleEvent();
- eventDate.clear();
- eventDate.set(2011, i, i, i, 0); // yy mm dd
- event.getHeaders().put("timestamp",
- String.valueOf(eventDate.getTimeInMillis()));
- event.getHeaders().put("hostname", "Host" + i);
- String body = "Test." + i + "." + j;
- event.setBody(body.getBytes());
- bodies.add(body);
- channel.put(event);
- totalEvents++;
- }
- txn.commit();
- txn.close();
-
- // execute sink to process the events
- sink.process();
- }
-
- sink.stop();
-
- // loop through all the files generated and check their contains
- FileStatus[] dirStat = fs.listStatus(dirPath);
- Path[] fList = FileUtil.stat2Paths(dirStat);
-
- // check that the roll happened correctly for the given data
- long expectedFiles = totalEvents / rollCount;
- if (totalEvents % rollCount > 0) expectedFiles++;
- Assert.assertEquals("num files wrong, found: " +
- Lists.newArrayList(fList), expectedFiles, fList.length);
- verifyOutputTextFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
- }
-
- @Test
- public void testAvroAppend() throws InterruptedException, LifecycleException,
- EventDeliveryException, IOException {
-
- LOG.debug("Starting...");
- final long rollCount = 3;
- final long batchSize = 2;
- final String fileName = "FlumeData";
- String newPath = testPath + "/singleTextBucket";
- int totalEvents = 0;
- int i = 1, j = 1;
-
- // clear the test directory
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(newPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
-
- Context context = new Context();
-
- // context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");
- context.put("hdfs.path", newPath);
- context.put("hdfs.filePrefix", fileName);
- context.put("hdfs.rollCount", String.valueOf(rollCount));
- context.put("hdfs.batchSize", String.valueOf(batchSize));
- context.put("hdfs.writeFormat", "Text");
- context.put("hdfs.fileType", "DataStream");
- context.put("serializer", "AVRO_EVENT");
-
- Configurables.configure(sink, context);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, context);
-
- sink.setChannel(channel);
- sink.start();
-
- Calendar eventDate = Calendar.getInstance();
- List<String> bodies = Lists.newArrayList();
-
- // push the event batches into channel
- for (i = 1; i < 4; i++) {
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (j = 1; j <= batchSize; j++) {
- Event event = new SimpleEvent();
- eventDate.clear();
- eventDate.set(2011, i, i, i, 0); // yy mm dd
- event.getHeaders().put("timestamp",
- String.valueOf(eventDate.getTimeInMillis()));
- event.getHeaders().put("hostname", "Host" + i);
- String body = "Test." + i + "." + j;
- event.setBody(body.getBytes());
- bodies.add(body);
- channel.put(event);
- totalEvents++;
- }
- txn.commit();
- txn.close();
-
- // execute sink to process the events
- sink.process();
- }
-
- sink.stop();
-
- // loop through all the files generated and check their contains
- FileStatus[] dirStat = fs.listStatus(dirPath);
- Path[] fList = FileUtil.stat2Paths(dirStat);
-
- // check that the roll happened correctly for the given data
- long expectedFiles = totalEvents / rollCount;
- if (totalEvents % rollCount > 0) expectedFiles++;
- Assert.assertEquals("num files wrong, found: " +
- Lists.newArrayList(fList), expectedFiles, fList.length);
- verifyOutputAvroFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
- }
-
- @Test
- public void testSimpleAppend() throws InterruptedException,
- LifecycleException, EventDeliveryException, IOException {
-
- LOG.debug("Starting...");
- final String fileName = "FlumeData";
- final long rollCount = 5;
- final long batchSize = 2;
- final int numBatches = 4;
- String newPath = testPath + "/singleBucket";
- int totalEvents = 0;
- int i = 1, j = 1;
-
- // clear the test directory
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(newPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
-
- Context context = new Context();
-
- context.put("hdfs.path", newPath);
- context.put("hdfs.filePrefix", fileName);
- context.put("hdfs.rollCount", String.valueOf(rollCount));
- context.put("hdfs.batchSize", String.valueOf(batchSize));
-
- Configurables.configure(sink, context);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, context);
-
- sink.setChannel(channel);
- sink.start();
-
- Calendar eventDate = Calendar.getInstance();
- List<String> bodies = Lists.newArrayList();
-
- // push the event batches into channel
- for (i = 1; i < numBatches; i++) {
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (j = 1; j <= batchSize; j++) {
- Event event = new SimpleEvent();
- eventDate.clear();
- eventDate.set(2011, i, i, i, 0); // yy mm dd
- event.getHeaders().put("timestamp",
- String.valueOf(eventDate.getTimeInMillis()));
- event.getHeaders().put("hostname", "Host" + i);
- String body = "Test." + i + "." + j;
- event.setBody(body.getBytes());
- bodies.add(body);
- channel.put(event);
- totalEvents++;
- }
- txn.commit();
- txn.close();
-
- // execute sink to process the events
- sink.process();
- }
-
- sink.stop();
-
- // loop through all the files generated and check their contains
- FileStatus[] dirStat = fs.listStatus(dirPath);
- Path[] fList = FileUtil.stat2Paths(dirStat);
-
- // check that the roll happened correctly for the given data
- long expectedFiles = totalEvents / rollCount;
- if (totalEvents % rollCount > 0) expectedFiles++;
- Assert.assertEquals("num files wrong, found: " +
- Lists.newArrayList(fList), expectedFiles, fList.length);
- verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
- }
-
- @Test
- public void testSimpleAppendLocalTime()
- throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
- final long currentTime = System.currentTimeMillis();
- Clock clk = new Clock() {
- @Override
- public long currentTimeMillis() {
- return currentTime;
- }
- };
-
- LOG.debug("Starting...");
- final String fileName = "FlumeData";
- final long rollCount = 5;
- final long batchSize = 2;
- final int numBatches = 4;
- String newPath = testPath + "/singleBucket/%s" ;
- String expectedPath = testPath + "/singleBucket/" +
- String.valueOf(currentTime / 1000);
- int totalEvents = 0;
- int i = 1, j = 1;
-
- // clear the test directory
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(expectedPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
-
- Context context = new Context();
-
- context.put("hdfs.path", newPath);
- context.put("hdfs.filePrefix", fileName);
- context.put("hdfs.rollCount", String.valueOf(rollCount));
- context.put("hdfs.batchSize", String.valueOf(batchSize));
- context.put("hdfs.useLocalTimeStamp", String.valueOf(true));
-
- Configurables.configure(sink, context);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, context);
-
- sink.setChannel(channel);
- sink.setBucketClock(clk);
- sink.start();
-
- Calendar eventDate = Calendar.getInstance();
- List<String> bodies = Lists.newArrayList();
-
- // push the event batches into channel
- for (i = 1; i < numBatches; i++) {
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (j = 1; j <= batchSize; j++) {
- Event event = new SimpleEvent();
- eventDate.clear();
- eventDate.set(2011, i, i, i, 0); // yy mm dd
- event.getHeaders().put("timestamp",
- String.valueOf(eventDate.getTimeInMillis()));
- event.getHeaders().put("hostname", "Host" + i);
- String body = "Test." + i + "." + j;
- event.setBody(body.getBytes());
- bodies.add(body);
- channel.put(event);
- totalEvents++;
- }
- txn.commit();
- txn.close();
-
- // execute sink to process the events
- sink.process();
- }
-
- sink.stop();
-
- // loop through all the files generated and check their contains
- FileStatus[] dirStat = fs.listStatus(dirPath);
- Path[] fList = FileUtil.stat2Paths(dirStat);
-
- // check that the roll happened correctly for the given data
- long expectedFiles = totalEvents / rollCount;
- if (totalEvents % rollCount > 0) expectedFiles++;
- Assert.assertEquals("num files wrong, found: " +
- Lists.newArrayList(fList), expectedFiles, fList.length);
- verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
- // The clock in bucketpath is static, so restore the real clock
- sink.setBucketClock(new SystemClock());
- }
-
- @Test
- public void testAppend() throws InterruptedException, LifecycleException,
- EventDeliveryException, IOException {
-
- LOG.debug("Starting...");
- final long rollCount = 3;
- final long batchSize = 2;
- final String fileName = "FlumeData";
-
- // clear the test directory
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(testPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
-
- Context context = new Context();
-
- context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");
- context.put("hdfs.timeZone", "UTC");
- context.put("hdfs.filePrefix", fileName);
- context.put("hdfs.rollCount", String.valueOf(rollCount));
- context.put("hdfs.batchSize", String.valueOf(batchSize));
-
- Configurables.configure(sink, context);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, context);
-
- sink.setChannel(channel);
- sink.start();
-
- Calendar eventDate = Calendar.getInstance();
- List<String> bodies = Lists.newArrayList();
- // push the event batches into channel
- for (int i = 1; i < 4; i++) {
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (int j = 1; j <= batchSize; j++) {
- Event event = new SimpleEvent();
- eventDate.clear();
- eventDate.set(2011, i, i, i, 0); // yy mm dd
- event.getHeaders().put("timestamp",
- String.valueOf(eventDate.getTimeInMillis()));
- event.getHeaders().put("hostname", "Host" + i);
- String body = "Test." + i + "." + j;
- event.setBody(body.getBytes());
- bodies.add(body);
- channel.put(event);
- }
- txn.commit();
- txn.close();
-
- // execute sink to process the events
- sink.process();
- }
-
- sink.stop();
- verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
- }
-
- // inject fault and make sure that the txn is rolled back and retried
- @Test
- public void testBadSimpleAppend() throws InterruptedException,
- LifecycleException, EventDeliveryException, IOException {
-
- LOG.debug("Starting...");
- final String fileName = "FlumeData";
- final long rollCount = 5;
- final long batchSize = 2;
- final int numBatches = 4;
- String newPath = testPath + "/singleBucket";
- int totalEvents = 0;
- int i = 1, j = 1;
-
- HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
- sink = new HDFSEventSink(badWriterFactory);
-
- // clear the test directory
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(newPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
-
- Context context = new Context();
-
- context.put("hdfs.path", newPath);
- context.put("hdfs.filePrefix", fileName);
- context.put("hdfs.rollCount", String.valueOf(rollCount));
- context.put("hdfs.batchSize", String.valueOf(batchSize));
- context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
-
- Configurables.configure(sink, context);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, context);
-
- sink.setChannel(channel);
- sink.start();
-
- Calendar eventDate = Calendar.getInstance();
-
- List<String> bodies = Lists.newArrayList();
- // push the event batches into channel
- for (i = 1; i < numBatches; i++) {
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (j = 1; j <= batchSize; j++) {
- Event event = new SimpleEvent();
- eventDate.clear();
- eventDate.set(2011, i, i, i, 0); // yy mm dd
- event.getHeaders().put("timestamp",
- String.valueOf(eventDate.getTimeInMillis()));
- event.getHeaders().put("hostname", "Host" + i);
-
- String body = "Test." + i + "." + j;
- event.setBody(body.getBytes());
- bodies.add(body);
- // inject fault
- if ((totalEvents % 30) == 1) {
- event.getHeaders().put("fault-once", "");
- }
- channel.put(event);
- totalEvents++;
- }
- txn.commit();
- txn.close();
-
- LOG.info("Process events: " + sink.process());
- }
- LOG.info("Process events to end of transaction max: " + sink.process());
- LOG.info("Process events to injected fault: " + sink.process());
- LOG.info("Process events remaining events: " + sink.process());
- sink.stop();
- verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
-
- SinkCounter sc = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
- Assert.assertEquals(1, sc.getEventWriteFail());
- }
-
-
- private List<String> getAllFiles(String input) {
- List<String> output = Lists.newArrayList();
- File dir = new File(input);
- if (dir.isFile()) {
- output.add(dir.getAbsolutePath());
- } else if (dir.isDirectory()) {
- for (String file : dir.list()) {
- File subDir = new File(dir, file);
- output.addAll(getAllFiles(subDir.getAbsolutePath()));
- }
- }
- return output;
- }
-
- private void verifyOutputSequenceFiles(FileSystem fs, Configuration conf, String dir,
- String prefix, List<String> bodies) throws IOException {
- int found = 0;
- int expected = bodies.size();
- for (String outputFile : getAllFiles(dir)) {
- String name = (new File(outputFile)).getName();
- if (name.startsWith(prefix)) {
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(outputFile), conf);
- LongWritable key = new LongWritable();
- BytesWritable value = new BytesWritable();
- while (reader.next(key, value)) {
- String body = new String(value.getBytes(), 0, value.getLength());
- if (bodies.contains(body)) {
- LOG.debug("Found event body: {}", body);
- bodies.remove(body);
- found++;
- }
- }
- reader.close();
- }
- }
- if (!bodies.isEmpty()) {
- for (String body : bodies) {
- LOG.error("Never found event body: {}", body);
- }
- }
- Assert.assertTrue("Found = " + found + ", Expected = " +
- expected + ", Left = " + bodies.size() + " " + bodies,
- bodies.size() == 0);
-
- }
-
- private void verifyOutputTextFiles(FileSystem fs, Configuration conf, String dir, String prefix,
- List<String> bodies) throws IOException {
- int found = 0;
- int expected = bodies.size();
- for (String outputFile : getAllFiles(dir)) {
- String name = (new File(outputFile)).getName();
- if (name.startsWith(prefix)) {
- FSDataInputStream input = fs.open(new Path(outputFile));
- BufferedReader reader = new BufferedReader(new InputStreamReader(input));
- String body = null;
- while ((body = reader.readLine()) != null) {
- bodies.remove(body);
- found++;
- }
- reader.close();
- }
- }
- Assert.assertTrue("Found = " + found + ", Expected = " +
- expected + ", Left = " + bodies.size() + " " + bodies,
- bodies.size() == 0);
-
- }
-
- private void verifyOutputAvroFiles(FileSystem fs, Configuration conf, String dir, String prefix,
- List<String> bodies) throws IOException {
- int found = 0;
- int expected = bodies.size();
- for (String outputFile : getAllFiles(dir)) {
- String name = (new File(outputFile)).getName();
- if (name.startsWith(prefix)) {
- FSDataInputStream input = fs.open(new Path(outputFile));
- DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
- DataFileStream<GenericRecord> avroStream =
- new DataFileStream<GenericRecord>(input, reader);
- GenericRecord record = new GenericData.Record(avroStream.getSchema());
- while (avroStream.hasNext()) {
- avroStream.next(record);
- ByteBuffer body = (ByteBuffer) record.get("body");
- CharsetDecoder decoder = Charsets.UTF_8.newDecoder();
- String bodyStr = decoder.decode(body).toString();
- LOG.debug("Removing event: {}", bodyStr);
- bodies.remove(bodyStr);
- found++;
- }
- avroStream.close();
- input.close();
- }
- }
- Assert.assertTrue("Found = " + found + ", Expected = " +
- expected + ", Left = " + bodies.size() + " " + bodies,
- bodies.size() == 0);
- }
-
- /**
- * Ensure that when a write throws an IOException we are
- * able to continue to progress in the next process() call.
- * This relies on Transactional rollback semantics for durability and
- * the behavior of the BucketWriter class of close()ing upon IOException.
- */
- @Test
- public void testCloseReopen()
- throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
-
- LOG.debug("Starting...");
- final int numBatches = 4;
- final String fileName = "FlumeData";
- final long rollCount = 5;
- final long batchSize = 2;
- String newPath = testPath + "/singleBucket";
- int i = 1, j = 1;
-
- HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
- sink = new HDFSEventSink(badWriterFactory);
-
- // clear the test directory
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(newPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
-
- Context context = new Context();
-
- context.put("hdfs.path", newPath);
- context.put("hdfs.filePrefix", fileName);
- context.put("hdfs.rollCount", String.valueOf(rollCount));
- context.put("hdfs.batchSize", String.valueOf(batchSize));
- context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
-
- Configurables.configure(sink, context);
-
- MemoryChannel channel = new MemoryChannel();
- Configurables.configure(channel, new Context());
-
- sink.setChannel(channel);
- sink.start();
-
- Calendar eventDate = Calendar.getInstance();
- List<String> bodies = Lists.newArrayList();
- // push the event batches into channel
- for (i = 1; i < numBatches; i++) {
- channel.getTransaction().begin();
- try {
- for (j = 1; j <= batchSize; j++) {
- Event event = new SimpleEvent();
- eventDate.clear();
- eventDate.set(2011, i, i, i, 0); // yy mm dd
- event.getHeaders().put("timestamp",
- String.valueOf(eventDate.getTimeInMillis()));
- event.getHeaders().put("hostname", "Host" + i);
- String body = "Test." + i + "." + j;
- event.setBody(body.getBytes());
- bodies.add(body);
- // inject fault
- event.getHeaders().put("fault-until-reopen", "");
- channel.put(event);
- }
- channel.getTransaction().commit();
- } finally {
- channel.getTransaction().close();
- }
- LOG.info("execute sink to process the events: " + sink.process());
- }
- LOG.info("clear any events pending due to errors: " + sink.process());
- sink.stop();
-
- verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
-
- SinkCounter sc = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
- Assert.assertEquals(1, sc.getEventWriteFail());
- }
-
- /**
- * Test that the old bucket writer is closed at the end of rollInterval and
- * a new one is used for the next set of events.
- */
- @Test
- public void testCloseReopenOnRollTime()
- throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
-
- LOG.debug("Starting...");
- final int numBatches = 4;
- final String fileName = "FlumeData";
- final long batchSize = 2;
- String newPath = testPath + "/singleBucket";
- int i = 1, j = 1;
-
- HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
- sink = new HDFSEventSink(badWriterFactory);
-
- // clear the test directory
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(newPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
-
- Context context = new Context();
-
- context.put("hdfs.path", newPath);
- context.put("hdfs.filePrefix", fileName);
- context.put("hdfs.rollCount", String.valueOf(0));
- context.put("hdfs.rollSize", String.valueOf(0));
- context.put("hdfs.rollInterval", String.valueOf(2));
- context.put("hdfs.batchSize", String.valueOf(batchSize));
- context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
-
- Configurables.configure(sink, context);
-
- MemoryChannel channel = new MemoryChannel();
- Configurables.configure(channel, new Context());
-
- sink.setChannel(channel);
- sink.start();
-
- Calendar eventDate = Calendar.getInstance();
- List<String> bodies = Lists.newArrayList();
- // push the event batches into channel
- for (i = 1; i < numBatches; i++) {
- channel.getTransaction().begin();
- try {
- for (j = 1; j <= batchSize; j++) {
- Event event = new SimpleEvent();
- eventDate.clear();
- eventDate.set(2011, i, i, i, 0); // yy mm dd
- event.getHeaders().put("timestamp",
- String.valueOf(eventDate.getTimeInMillis()));
- event.getHeaders().put("hostname", "Host" + i);
- String body = "Test." + i + "." + j;
- event.setBody(body.getBytes());
- bodies.add(body);
- // inject fault
- event.getHeaders().put("count-check", "");
- channel.put(event);
- }
- channel.getTransaction().commit();
- } finally {
- channel.getTransaction().close();
- }
- LOG.info("execute sink to process the events: " + sink.process());
- // Make sure the first file gets rolled due to rollTimeout.
- if (i == 1) {
- Thread.sleep(2001);
- }
- }
- LOG.info("clear any events pending due to errors: " + sink.process());
- sink.stop();
-
- Assert.assertTrue(badWriterFactory.openCount.get() >= 2);
- LOG.info("Total number of bucket writers opened: {}",
- badWriterFactory.openCount.get());
- verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName,
- bodies);
- }
-
- /**
- * Test that a close due to roll interval removes the bucketwriter from
- * sfWriters map.
- */
- @Test
- public void testCloseRemovesFromSFWriters()
- throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
-
- LOG.debug("Starting...");
- final String fileName = "FlumeData";
- final long batchSize = 2;
- String newPath = testPath + "/singleBucket";
- int i = 1, j = 1;
-
- HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
- sink = new HDFSEventSink(badWriterFactory);
-
- // clear the test directory
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(newPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
-
- Context context = new Context();
-
- context.put("hdfs.path", newPath);
- context.put("hdfs.filePrefix", fileName);
- context.put("hdfs.rollCount", String.valueOf(0));
- context.put("hdfs.rollSize", String.valueOf(0));
- context.put("hdfs.rollInterval", String.valueOf(1));
- context.put("hdfs.batchSize", String.valueOf(batchSize));
- context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
- String expectedLookupPath = newPath + "/FlumeData";
-
- Configurables.configure(sink, context);
-
- MemoryChannel channel = new MemoryChannel();
- Configurables.configure(channel, new Context());
-
- sink.setChannel(channel);
- sink.start();
-
- Calendar eventDate = Calendar.getInstance();
- List<String> bodies = Lists.newArrayList();
- // push the event batches into channel
- channel.getTransaction().begin();
- try {
- for (j = 1; j <= 2 * batchSize; j++) {
- Event event = new SimpleEvent();
- eventDate.clear();
- eventDate.set(2011, i, i, i, 0); // yy mm dd
- event.getHeaders().put("timestamp",
- String.valueOf(eventDate.getTimeInMillis()));
- event.getHeaders().put("hostname", "Host" + i);
- String body = "Test." + i + "." + j;
- event.setBody(body.getBytes());
- bodies.add(body);
- // inject fault
- event.getHeaders().put("count-check", "");
- channel.put(event);
- }
- channel.getTransaction().commit();
- } finally {
- channel.getTransaction().close();
- }
- LOG.info("execute sink to process the events: " + sink.process());
- Assert.assertTrue(sink.getSfWriters().containsKey(expectedLookupPath));
- // Make sure the first file gets rolled due to rollTimeout.
- Thread.sleep(2001);
- Assert.assertFalse(sink.getSfWriters().containsKey(expectedLookupPath));
- LOG.info("execute sink to process the events: " + sink.process());
- // A new bucket writer should have been created for this bucket. So
- // sfWriters map should not have the same key again.
- Assert.assertTrue(sink.getSfWriters().containsKey(expectedLookupPath));
- sink.stop();
-
- LOG.info("Total number of bucket writers opened: {}",
- badWriterFactory.openCount.get());
- verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName,
- bodies);
- }
-
-
-
- /*
- * append using slow sink writer.
- * verify that the process returns backoff due to timeout
- */
- @Test
- public void testSlowAppendFailure() throws InterruptedException,
- LifecycleException, EventDeliveryException, IOException {
-
- LOG.debug("Starting...");
- final String fileName = "FlumeData";
- final long rollCount = 5;
- final long batchSize = 2;
- final int numBatches = 2;
- String newPath = testPath + "/singleBucket";
- int i = 1, j = 1;
-
- // clear the test directory
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(newPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
-
- // create HDFS sink with slow writer
- HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
- sink = new HDFSEventSink(badWriterFactory);
-
- Context context = new Context();
- context.put("hdfs.path", newPath);
- context.put("hdfs.filePrefix", fileName);
- context.put("hdfs.rollCount", String.valueOf(rollCount));
- context.put("hdfs.batchSize", String.valueOf(batchSize));
- context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
- context.put("hdfs.callTimeout", Long.toString(1000));
- Configurables.configure(sink, context);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, context);
-
- sink.setChannel(channel);
- sink.start();
-
- Calendar eventDate = Calendar.getInstance();
-
- // push the event batches into channel
- for (i = 0; i < numBatches; i++) {
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (j = 1; j <= batchSize; j++) {
- Event event = new SimpleEvent();
- eventDate.clear();
- eventDate.set(2011, i, i, i, 0); // yy mm dd
- event.getHeaders().put("timestamp",
- String.valueOf(eventDate.getTimeInMillis()));
- event.getHeaders().put("hostname", "Host" + i);
- event.getHeaders().put("slow", "1500");
- event.setBody(("Test." + i + "." + j).getBytes());
- channel.put(event);
- }
- txn.commit();
- txn.close();
-
- // execute sink to process the events
- Status satus = sink.process();
-
- // verify that the append returned backoff due to timeotu
- Assert.assertEquals(satus, Status.BACKOFF);
- }
-
- sink.stop();
-
- SinkCounter sc = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
- Assert.assertEquals(2, sc.getEventWriteFail());
- }
-
- /*
- * append using slow sink writer with specified append timeout
- * verify that the data is written correctly to files
- */
- private void slowAppendTestHelper(long appendTimeout)
- throws InterruptedException, IOException, LifecycleException, EventDeliveryException,
- IOException {
- final String fileName = "FlumeData";
- final long rollCount = 5;
- final long batchSize = 2;
- final int numBatches = 2;
- String newPath = testPath + "/singleBucket";
- int totalEvents = 0;
- int i = 1, j = 1;
-
- // clear the test directory
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(newPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
-
- // create HDFS sink with slow writer
- HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
- sink = new HDFSEventSink(badWriterFactory);
-
- Context context = new Context();
- context.put("hdfs.path", newPath);
- context.put("hdfs.filePrefix", fileName);
- context.put("hdfs.rollCount", String.valueOf(rollCount));
- context.put("hdfs.batchSize", String.valueOf(batchSize));
- context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
- context.put("hdfs.appendTimeout", String.valueOf(appendTimeout));
- Configurables.configure(sink, context);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, context);
-
- sink.setChannel(channel);
- sink.start();
-
- Calendar eventDate = Calendar.getInstance();
- List<String> bodies = Lists.newArrayList();
- // push the event batches into channel
- for (i = 0; i < numBatches; i++) {
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (j = 1; j <= batchSize; j++) {
- Event event = new SimpleEvent();
- eventDate.clear();
- eventDate.set(2011, i, i, i, 0); // yy mm dd
- event.getHeaders().put("timestamp",
- String.valueOf(eventDate.getTimeInMillis()));
- event.getHeaders().put("hostname", "Host" + i);
- event.getHeaders().put("slow", "1500");
- String body = "Test." + i + "." + j;
- event.setBody(body.getBytes());
- bodies.add(body);
- channel.put(event);
- totalEvents++;
- }
- txn.commit();
- txn.close();
-
- // execute sink to process the events
- sink.process();
- }
-
- sink.stop();
-
- // loop through all the files generated and check their contains
- FileStatus[] dirStat = fs.listStatus(dirPath);
- Path[] fList = FileUtil.stat2Paths(dirStat);
-
- // check that the roll happened correctly for the given data
- // Note that we'll end up with two files with only a head
- long expectedFiles = totalEvents / rollCount;
- if (totalEvents % rollCount > 0) expectedFiles++;
- Assert.assertEquals("num files wrong, found: " +
- Lists.newArrayList(fList), expectedFiles, fList.length);
- verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
- }
-
- /*
- * append using slow sink writer with long append timeout
- * verify that the data is written correctly to files
- */
- @Test
- public void testSlowAppendWithLongTimeout() throws InterruptedException,
- LifecycleException, EventDeliveryException, IOException {
- LOG.debug("Starting...");
- slowAppendTestHelper(3000);
- }
-
- /*
- * append using slow sink writer with no timeout to make append
- * synchronous. Verify that the data is written correctly to files
- */
- @Test
- public void testSlowAppendWithoutTimeout() throws InterruptedException,
- LifecycleException, EventDeliveryException, IOException {
- LOG.debug("Starting...");
- slowAppendTestHelper(0);
- }
- @Test
- public void testCloseOnIdle() throws IOException, EventDeliveryException, InterruptedException {
- String hdfsPath = testPath + "/idleClose";
-
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(hdfsPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
- Context context = new Context();
- context.put("hdfs.path", hdfsPath);
- /*
- * All three rolling methods are disabled so the only
- * way a file can roll is through the idle timeout.
- */
- context.put("hdfs.rollCount", "0");
- context.put("hdfs.rollSize", "0");
- context.put("hdfs.rollInterval", "0");
- context.put("hdfs.batchSize", "2");
- context.put("hdfs.idleTimeout", "1");
- Configurables.configure(sink, context);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, context);
-
- sink.setChannel(channel);
- sink.start();
-
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (int i = 0; i < 10; i++) {
- Event event = new SimpleEvent();
- event.setBody(("test event " + i).getBytes());
- channel.put(event);
- }
- txn.commit();
- txn.close();
-
- sink.process();
- sink.process();
- Thread.sleep(1101);
- // previous file should have timed out now
- // this can throw BucketClosedException(from the bucketWriter having
- // closed),this is not an issue as the sink will retry and get a fresh
- // bucketWriter so long as the onClose handler properly removes
- // bucket writers that were closed.
- sink.process();
- sink.process();
- Thread.sleep(500); // shouldn't be enough for a timeout to occur
- sink.process();
- sink.process();
- sink.stop();
- FileStatus[] dirStat = fs.listStatus(dirPath);
- Path[] fList = FileUtil.stat2Paths(dirStat);
- Assert.assertEquals("Incorrect content of the directory " + StringUtils.join(fList, ","),
- 2, fList.length);
- Assert.assertTrue(!fList[0].getName().endsWith(".tmp") &&
- !fList[1].getName().endsWith(".tmp"));
- fs.close();
- }
-
- /**
- * This test simulates what happens when a batch of events is written to a compressed sequence
- * file (and thus hsync'd to hdfs) but the file is not yet closed.
- *
- * When this happens, the data that we wrote should still be readable.
- */
- @Test
- public void testBlockCompressSequenceFileWriterSync() throws IOException, EventDeliveryException {
- String hdfsPath = testPath + "/sequenceFileWriterSync";
- FileSystem fs = FileSystem.get(new Configuration());
- // Since we are reading a partial file we don't want to use checksums
- fs.setVerifyChecksum(false);
- fs.setWriteChecksum(false);
-
- // Compression codecs that don't require native hadoop libraries
- String [] codecs = {"BZip2Codec", "DeflateCodec"};
-
- for (String codec : codecs) {
- sequenceFileWriteAndVerifyEvents(fs, hdfsPath, codec, Collections.singletonList(
- "single-event"
- ));
-
- sequenceFileWriteAndVerifyEvents(fs, hdfsPath, codec, Arrays.asList(
- "multiple-events-1",
- "multiple-events-2",
- "multiple-events-3",
- "multiple-events-4",
- "multiple-events-5"
- ));
- }
-
- fs.close();
- }
-
- private void sequenceFileWriteAndVerifyEvents(FileSystem fs, String hdfsPath, String codec,
- Collection<String> eventBodies)
- throws IOException, EventDeliveryException {
- Path dirPath = new Path(hdfsPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
-
- Context context = new Context();
- context.put("hdfs.path", hdfsPath);
- // Ensure the file isn't closed and rolled
- context.put("hdfs.rollCount", String.valueOf(eventBodies.size() + 1));
- context.put("hdfs.rollSize", "0");
- context.put("hdfs.rollInterval", "0");
- context.put("hdfs.batchSize", "1");
- context.put("hdfs.fileType", "SequenceFile");
- context.put("hdfs.codeC", codec);
- context.put("hdfs.writeFormat", "Writable");
- Configurables.configure(sink, context);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, context);
-
- sink.setChannel(channel);
- sink.start();
-
- for (String eventBody : eventBodies) {
- Transaction txn = channel.getTransaction();
- txn.begin();
-
- Event event = new SimpleEvent();
- event.setBody(eventBody.getBytes());
- channel.put(event);
-
- txn.commit();
- txn.close();
-
- sink.process();
- }
-
- // Sink is _not_ closed. The file should remain open but
- // the data written should be visible to readers via sync + hflush
- FileStatus[] dirStat = fs.listStatus(dirPath);
- Path[] paths = FileUtil.stat2Paths(dirStat);
-
- Assert.assertEquals(1, paths.length);
-
- SequenceFile.Reader reader =
- new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.stream(fs.open(paths[0])));
- LongWritable key = new LongWritable();
- BytesWritable value = new BytesWritable();
-
- for (String eventBody : eventBodies) {
- Assert.assertTrue(reader.next(key, value));
- Assert.assertArrayEquals(eventBody.getBytes(), value.copyBytes());
- }
-
- Assert.assertFalse(reader.next(key, value));
- }
-
- private Context getContextForRetryTests() {
- Context context = new Context();
-
- context.put("hdfs.path", testPath + "/%{retryHeader}");
- context.put("hdfs.filePrefix", "test");
- context.put("hdfs.batchSize", String.valueOf(100));
- context.put("hdfs.fileType", "DataStream");
- context.put("hdfs.serializer", "text");
- context.put("hdfs.closeTries","3");
- context.put("hdfs.rollCount", "1");
- context.put("hdfs.retryInterval", "1");
- return context;
- }
-
- @Test
- public void testBadConfigurationForRetryIntervalZero() throws Exception {
- Context context = getContextForRetryTests();
- context.put("hdfs.retryInterval", "0");
-
- Configurables.configure(sink, context);
- Assert.assertEquals(1, sink.getTryCount());
- }
-
- @Test
- public void testBadConfigurationForRetryIntervalNegative() throws Exception {
- Context context = getContextForRetryTests();
- context.put("hdfs.retryInterval", "-1");
-
- Configurables.configure(sink, context);
- Assert.assertEquals(1, sink.getTryCount());
- }
-
- @Test
- public void testBadConfigurationForRetryCountZero() throws Exception {
- Context context = getContextForRetryTests();
- context.put("hdfs.closeTries" ,"0");
-
- Configurables.configure(sink, context);
- Assert.assertEquals(Integer.MAX_VALUE, sink.getTryCount());
- }
-
- @Test
- public void testBadConfigurationForRetryCountNegative() throws Exception {
- Context context = getContextForRetryTests();
- context.put("hdfs.closeTries" ,"-4");
-
- Configurables.configure(sink, context);
- Assert.assertEquals(Integer.MAX_VALUE, sink.getTryCount());
- }
-
- @Test
- public void testRetryRename()
- throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
- testRetryRename(true);
- testRetryRename(false);
- }
-
- private void testRetryRename(boolean closeSucceed)
- throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
- LOG.debug("Starting...");
- String newPath = testPath + "/retryBucket";
-
- // clear the test directory
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- Path dirPath = new Path(newPath);
- fs.delete(dirPath, true);
- fs.mkdirs(dirPath);
- MockFileSystem mockFs = new MockFileSystem(fs, 6, closeSucceed);
-
- Context context = getContextForRetryTests();
- Configurables.configure(sink, context);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, context);
-
- sink.setChannel(channel);
- sink.setMockFs(mockFs);
- HDFSWriter hdfsWriter = new MockDataStream(mockFs);
- hdfsWriter.configure(context);
- sink.setMockWriter(hdfsWriter);
- sink.start();
-
- // push the event batches into channel
- for (int i = 0; i < 2; i++) {
- Transaction txn = channel.getTransaction();
- txn.begin();
- Map<String, String> hdr = Maps.newHashMap();
- hdr.put("retryHeader", "v1");
-
- channel.put(EventBuilder.withBody("random".getBytes(), hdr));
- txn.commit();
- txn.close();
-
- // execute sink to process the events
- sink.process();
- }
- // push the event batches into channel
- for (int i = 0; i < 2; i++) {
- Transaction txn = channel.getTransaction();
- txn.begin();
- Map<String, String> hdr = Maps.newHashMap();
- hdr.put("retryHeader", "v2");
- channel.put(EventBuilder.withBody("random".getBytes(), hdr));
- txn.commit();
- txn.close();
- // execute sink to process the events
- sink.process();
- }
-
- TimeUnit.SECONDS.sleep(5); //Sleep till all retries are done.
-
- Collection<BucketWriter> writers = sink.getSfWriters().values();
-
- int totalRenameAttempts = 0;
- for (BucketWriter writer : writers) {
- LOG.info("Rename tries = " + writer.renameTries.get());
- totalRenameAttempts += writer.renameTries.get();
- }
- // stop clears the sfWriters map, so we need to compute the
- // close tries count before stopping the sink.
- sink.stop();
- Assert.assertEquals(6, totalRenameAttempts);
-
- }
-
- /**
- * BucketWriter.append() can throw a BucketClosedException when called from
- * HDFSEventSink.process() due to a race condition between HDFSEventSink.process() and the
- * BucketWriter's close threads.
- * This test case tests whether if this happens the newly created BucketWriter will be flushed.
- * For more details see FLUME-3085
- */
- @Test
- public void testFlushedIfAppendFailedWithBucketClosedException() throws Exception {
- final Set<BucketWriter> bucketWriters = new HashSet<>();
- sink = new HDFSEventSink() {
- @Override
- BucketWriter initializeBucketWriter(String realPath, String realName, String lookupPath,
- HDFSWriter hdfsWriter, WriterCallback closeCallback) {
- BucketWriter bw = Mockito.spy(super.initializeBucketWriter(realPath, realName, lookupPath,
- hdfsWriter, closeCallback));
- try {
- // create mock BucketWriters where the first append() succeeds but the
- // the second call throws a BucketClosedException
- Mockito.doCallRealMethod()
- .doThrow(BucketClosedException.class)
- .when(bw).append(Mockito.any(Event.class));
- } catch (IOException | InterruptedException e) {
- Assert.fail("This shouldn't happen, as append() is called during mocking.");
- }
- bucketWriters.add(bw);
- return bw;
- }
- };
-
- Context context = new Context(ImmutableMap.of("hdfs.path", testPath));
- Configurables.configure(sink, context);
-
- Channel channel = Mockito.spy(new MemoryChannel());
- Configurables.configure(channel, new Context());
-
- final Iterator<Event> events = Iterators.forArray(
- EventBuilder.withBody("test1".getBytes()), EventBuilder.withBody("test2".getBytes()));
- Mockito.doAnswer(new Answer() {
- @Override
- public Object answer(InvocationOnMock invocation) throws Throwable {
- return events.hasNext() ? events.next() : null;
- }
- }).when(channel).take();
-
- sink.setChannel(channel);
- sink.start();
-
- sink.process();
-
- // channel.take() should have called 3 times (2 events + 1 null)
- Mockito.verify(channel, Mockito.times(3)).take();
-
- FileSystem fs = FileSystem.get(new Configuration());
- int fileCount = 0;
- for (RemoteIterator<LocatedFileStatus> i = fs.listFiles(new Path(testPath), false);
- i.hasNext(); i.next()) {
- fileCount++;
- }
- Assert.assertEquals(2, fileCount);
-
- Assert.assertEquals(2, bucketWriters.size());
- // It is expected that flush() method was called exactly once for every BucketWriter
- for (BucketWriter bw : bucketWriters) {
- Mockito.verify(bw, Mockito.times(1)).flush();
- }
-
- sink.stop();
- }
-
- @Test
- public void testChannelException() {
- LOG.debug("Starting...");
- Context context = new Context();
- context.put("hdfs.path", testPath);
- context.put("keep-alive", "0");
- Configurables.configure(sink, context);
- Channel channel = Mockito.mock(Channel.class);
- Mockito.when(channel.take()).thenThrow(new ChannelException("dummy"));
- Mockito.when(channel.getTransaction())
- .thenReturn(Mockito.mock(BasicTransactionSemantics.class));
- sink.setChannel(channel);
- sink.start();
- try {
- sink.process();
- } catch (EventDeliveryException e) {
- //
- }
- sink.stop();
-
- SinkCounter sc = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
- Assert.assertEquals(1, sc.getChannelReadFail());
- }
-
- @Test
- public void testEmptyInUseSuffix() {
- String inUseSuffixConf = "aaaa";
- Context context = new Context();
- context.put("hdfs.path", testPath);
- context.put("hdfs.inUseSuffix", inUseSuffixConf);
-
- //hdfs.emptyInUseSuffix not defined
- Configurables.configure(sink, context);
- String inUseSuffix = (String) Whitebox.getInternalState(sink, "inUseSuffix");
- Assert.assertEquals(inUseSuffixConf, inUseSuffix);
-
- context.put("hdfs.emptyInUseSuffix", "true");
- Configurables.configure(sink, context);
- inUseSuffix = (String) Whitebox.getInternalState(sink, "inUseSuffix");
- Assert.assertEquals("", inUseSuffix);
-
- context.put("hdfs.emptyInUseSuffix", "false");
- Configurables.configure(sink, context);
- inUseSuffix = (String) Whitebox.getInternalState(sink, "inUseSuffix");
- Assert.assertEquals(inUseSuffixConf, inUseSuffix);
- }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkDeadlock.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkDeadlock.java
deleted file mode 100644
index 1e1e8642c..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkDeadlock.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.primitives.Longs;
-import org.apache.flume.Channel;
-import org.apache.flume.ChannelSelector;
-import org.apache.flume.Context;
-import org.apache.flume.SinkProcessor;
-import org.apache.flume.SinkRunner;
-import org.apache.flume.channel.ChannelProcessor;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.channel.ReplicatingChannelSelector;
-import org.apache.flume.conf.Configurables;
-import org.apache.flume.sink.DefaultSinkProcessor;
-import org.apache.flume.source.SequenceGeneratorSource;
-
-import java.lang.management.ManagementFactory;
-import java.lang.management.ThreadMXBean;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-public class TestHDFSEventSinkDeadlock {
-
- public static void main(String... args) {
- HDFSEventSink sink = new HDFSEventSink();
- sink.setName("HDFSEventSink");
-
- Context context = new Context(ImmutableMap.of(
- "hdfs.path", "file:///tmp/flume-test/bucket-%t",
- "hdfs.filePrefix", "flumetest",
- "hdfs.rollInterval", "1",
- "hdfs.maxOpenFiles", "1",
- "hdfs.useLocalTimeStamp", "true"));
- Configurables.configure(sink, context);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, new Context());
-
- final SequenceGeneratorSource source = new SequenceGeneratorSource();
- Configurables.configure(source, new Context());
-
- ChannelSelector rcs = new ReplicatingChannelSelector();
- rcs.setChannels(Collections.singletonList(channel));
- source.setChannelProcessor(new ChannelProcessor(rcs));
-
- sink.setChannel(channel);
-
- channel.start();
- source.start();
-
- SinkProcessor sinkProcessor = new DefaultSinkProcessor();
- sinkProcessor.setSinks(Collections.singletonList(sink));
- SinkRunner sinkRunner = new SinkRunner();
- sinkRunner.setSink(sinkProcessor);
- sinkRunner.start();
-
- ScheduledExecutorService executor = Executors.newScheduledThreadPool(3);
-
- executor.execute(new Runnable() {
- @Override
- public void run() {
- int i = 0;
- while (true) {
- try {
- source.process();
- System.out.println(i++);
- if (i == 250) {
- System.out.println("No deadlock found after 250 iterations, exiting");
- System.exit(0);
- }
- Thread.sleep((long) (Math.random() * 100 + 950));
- } catch (Exception e) {
- //
- }
- }
- }
- });
-
- executor.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- ThreadMXBean bean = ManagementFactory.getThreadMXBean();
- long[] threadIds = bean.findDeadlockedThreads();
- if (threadIds != null) {
- System.out.println("Deadlocked threads found");
- printThreadStackTraces(threadIds);
- System.exit(1);
- }
- }
- }, 0, 1, TimeUnit.SECONDS);
- }
-
- private static void printThreadStackTraces(long[] threadIds) {
- Set<Long> threadIdSet = new HashSet<>(Longs.asList(threadIds));
- for (Thread th : Thread.getAllStackTraces().keySet()) {
- if (threadIdSet.contains(th.getId())) {
- System.out.println("Thread: " + th);
- for (StackTraceElement e : th.getStackTrace()) {
- System.out.println("\t" + e);
- }
- System.out.println("-----------------------------");
- }
- }
- }
-}
\ No newline at end of file
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkOnMiniCluster.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkOnMiniCluster.java
deleted file mode 100644
index 6dfac8a3f..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkOnMiniCluster.java
+++ /dev/null
@@ -1,607 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import com.google.common.base.Charsets;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
-import java.util.zip.GZIPInputStream;
-
-import com.google.common.base.Throwables;
-import org.apache.commons.io.FileUtils;
-import org.apache.flume.Context;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.Transaction;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.event.EventBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Unit tests that exercise HDFSEventSink on an actual instance of HDFS.
- * TODO: figure out how to unit-test Kerberos-secured HDFS.
- */
-public class TestHDFSEventSinkOnMiniCluster {
-
- private static final Logger logger =
- LoggerFactory.getLogger(TestHDFSEventSinkOnMiniCluster.class);
-
- private static final boolean KEEP_DATA = false;
- private static final String DFS_DIR = "target/test/dfs";
- private static final String TEST_BUILD_DATA_KEY = "test.build.data";
-
- private static MiniDFSCluster cluster = null;
- private static String oldTestBuildDataProp = null;
-
- @BeforeClass
- public static void setupClass() throws IOException {
- // set up data dir for HDFS
- File dfsDir = new File(DFS_DIR);
- if (!dfsDir.isDirectory()) {
- dfsDir.mkdirs();
- }
- // save off system prop to restore later
- oldTestBuildDataProp = System.getProperty(TEST_BUILD_DATA_KEY);
- System.setProperty(TEST_BUILD_DATA_KEY, DFS_DIR);
- }
-
- private static String getNameNodeURL(MiniDFSCluster cluster) {
- int nnPort = cluster.getNameNode().getNameNodeAddress().getPort();
- return "hdfs://localhost:" + nnPort;
- }
-
- /**
- * This is a very basic test that writes one event to HDFS and reads it back.
- */
- @Test
- public void simpleHDFSTest() throws EventDeliveryException, IOException {
- cluster = new MiniDFSCluster(new Configuration(), 1, true, null);
- cluster.waitActive();
-
- String outputDir = "/flume/simpleHDFSTest";
- Path outputDirPath = new Path(outputDir);
-
- logger.info("Running test with output dir: {}", outputDir);
-
- FileSystem fs = cluster.getFileSystem();
- // ensure output directory is empty
- if (fs.exists(outputDirPath)) {
- fs.delete(outputDirPath, true);
- }
-
- String nnURL = getNameNodeURL(cluster);
- logger.info("Namenode address: {}", nnURL);
-
- Context chanCtx = new Context();
- MemoryChannel channel = new MemoryChannel();
- channel.setName("simpleHDFSTest-mem-chan");
- channel.configure(chanCtx);
- channel.start();
-
- Context sinkCtx = new Context();
- sinkCtx.put("hdfs.path", nnURL + outputDir);
- sinkCtx.put("hdfs.fileType", HDFSWriterFactory.DataStreamType);
- sinkCtx.put("hdfs.batchSize", Integer.toString(1));
-
- HDFSEventSink sink = new HDFSEventSink();
- sink.setName("simpleHDFSTest-hdfs-sink");
- sink.configure(sinkCtx);
- sink.setChannel(channel);
- sink.start();
-
- // create an event
- String EVENT_BODY = "yarg!";
- channel.getTransaction().begin();
- try {
- channel.put(EventBuilder.withBody(EVENT_BODY, Charsets.UTF_8));
- channel.getTransaction().commit();
- } finally {
- channel.getTransaction().close();
- }
-
- // store event to HDFS
- sink.process();
-
- // shut down flume
- sink.stop();
- channel.stop();
-
- // verify that it's in HDFS and that its content is what we say it should be
- FileStatus[] statuses = fs.listStatus(outputDirPath);
- Assert.assertNotNull("No files found written to HDFS", statuses);
- Assert.assertEquals("Only one file expected", 1, statuses.length);
-
- for (FileStatus status : statuses) {
- Path filePath = status.getPath();
- logger.info("Found file on DFS: {}", filePath);
- FSDataInputStream stream = fs.open(filePath);
- BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
- String line = reader.readLine();
- logger.info("First line in file {}: {}", filePath, line);
- Assert.assertEquals(EVENT_BODY, line);
- }
-
- if (!KEEP_DATA) {
- fs.delete(outputDirPath, true);
- }
-
- cluster.shutdown();
- cluster = null;
- }
-
- /**
- * Writes two events in GZIP-compressed serialize.
- */
- @Test
- public void simpleHDFSGZipCompressedTest() throws EventDeliveryException, IOException {
- cluster = new MiniDFSCluster(new Configuration(), 1, true, null);
- cluster.waitActive();
-
- String outputDir = "/flume/simpleHDFSGZipCompressedTest";
- Path outputDirPath = new Path(outputDir);
-
- logger.info("Running test with output dir: {}", outputDir);
-
- FileSystem fs = cluster.getFileSystem();
- // ensure output directory is empty
- if (fs.exists(outputDirPath)) {
- fs.delete(outputDirPath, true);
- }
-
- String nnURL = getNameNodeURL(cluster);
- logger.info("Namenode address: {}", nnURL);
-
- Context chanCtx = new Context();
- MemoryChannel channel = new MemoryChannel();
- channel.setName("simpleHDFSTest-mem-chan");
- channel.configure(chanCtx);
- channel.start();
-
- Context sinkCtx = new Context();
- sinkCtx.put("hdfs.path", nnURL + outputDir);
- sinkCtx.put("hdfs.fileType", HDFSWriterFactory.CompStreamType);
- sinkCtx.put("hdfs.batchSize", Integer.toString(1));
- sinkCtx.put("hdfs.codeC", "gzip");
-
- HDFSEventSink sink = new HDFSEventSink();
- sink.setName("simpleHDFSTest-hdfs-sink");
- sink.configure(sinkCtx);
- sink.setChannel(channel);
- sink.start();
-
- // create an event
- String EVENT_BODY_1 = "yarg1";
- String EVENT_BODY_2 = "yarg2";
- channel.getTransaction().begin();
- try {
- channel.put(EventBuilder.withBody(EVENT_BODY_1, Charsets.UTF_8));
- channel.put(EventBuilder.withBody(EVENT_BODY_2, Charsets.UTF_8));
- channel.getTransaction().commit();
- } finally {
- channel.getTransaction().close();
- }
-
- // store event to HDFS
- sink.process();
-
- // shut down flume
- sink.stop();
- channel.stop();
-
- // verify that it's in HDFS and that its content is what we say it should be
- FileStatus[] statuses = fs.listStatus(outputDirPath);
- Assert.assertNotNull("No files found written to HDFS", statuses);
- Assert.assertEquals("Only one file expected", 1, statuses.length);
-
- for (FileStatus status : statuses) {
- Path filePath = status.getPath();
- logger.info("Found file on DFS: {}", filePath);
- FSDataInputStream stream = fs.open(filePath);
- BufferedReader reader = new BufferedReader(new InputStreamReader(
- new GZIPInputStream(stream)));
- String line = reader.readLine();
- logger.info("First line in file {}: {}", filePath, line);
- Assert.assertEquals(EVENT_BODY_1, line);
-
- // The rest of this test is commented-out (will fail) for 2 reasons:
- //
- // (1) At the time of this writing, Hadoop has a bug which causes the
- // non-native gzip implementation to create invalid gzip files when
- // finish() and resetState() are called. See HADOOP-8522.
- //
- // (2) Even if HADOOP-8522 is fixed, the JDK GZipInputStream is unable
- // to read multi-member (concatenated) gzip files. See this Sun bug:
- // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4691425
- //
- //line = reader.readLine();
- //logger.info("Second line in file {}: {}", filePath, line);
- //Assert.assertEquals(EVENT_BODY_2, line);
- }
-
- if (!KEEP_DATA) {
- fs.delete(outputDirPath, true);
- }
-
- cluster.shutdown();
- cluster = null;
- }
-
- /**
- * This is a very basic test that writes one event to HDFS and reads it back.
- */
- @Test
- public void underReplicationTest() throws EventDeliveryException,
- IOException {
- Configuration conf = new Configuration();
- conf.set("dfs.replication", String.valueOf(3));
- cluster = new MiniDFSCluster(conf, 3, true, null);
- cluster.waitActive();
-
- String outputDir = "/flume/underReplicationTest";
- Path outputDirPath = new Path(outputDir);
-
- logger.info("Running test with output dir: {}", outputDir);
-
- FileSystem fs = cluster.getFileSystem();
- // ensure output directory is empty
- if (fs.exists(outputDirPath)) {
- fs.delete(outputDirPath, true);
- }
-
- String nnURL = getNameNodeURL(cluster);
- logger.info("Namenode address: {}", nnURL);
-
- Context chanCtx = new Context();
- MemoryChannel channel = new MemoryChannel();
- channel.setName("simpleHDFSTest-mem-chan");
- channel.configure(chanCtx);
- channel.start();
-
- Context sinkCtx = new Context();
- sinkCtx.put("hdfs.path", nnURL + outputDir);
- sinkCtx.put("hdfs.fileType", HDFSWriterFactory.DataStreamType);
- sinkCtx.put("hdfs.batchSize", Integer.toString(1));
- sinkCtx.put("hdfs.retryInterval", "10"); //to speed up test
-
- HDFSEventSink sink = new HDFSEventSink();
- sink.setName("simpleHDFSTest-hdfs-sink");
- sink.configure(sinkCtx);
- sink.setChannel(channel);
- sink.start();
-
- // create an event
- channel.getTransaction().begin();
- try {
- channel.put(EventBuilder.withBody("yarg 1", Charsets.UTF_8));
- channel.put(EventBuilder.withBody("yarg 2", Charsets.UTF_8));
- channel.put(EventBuilder.withBody("yarg 3", Charsets.UTF_8));
- channel.put(EventBuilder.withBody("yarg 4", Charsets.UTF_8));
- channel.put(EventBuilder.withBody("yarg 5", Charsets.UTF_8));
- channel.put(EventBuilder.withBody("yarg 5", Charsets.UTF_8));
- channel.getTransaction().commit();
- } finally {
- channel.getTransaction().close();
- }
-
- // store events to HDFS
- logger.info("Running process(). Create new file.");
- sink.process(); // create new file;
- logger.info("Running process(). Same file.");
- sink.process();
-
- // kill a datanode
- logger.info("Killing datanode #1...");
- cluster.stopDataNode(0);
-
- // there is a race here.. the client may or may not notice that the
- // datanode is dead before it next sync()s.
- // so, this next call may or may not roll a new file.
-
- logger.info("Running process(). Create new file? (racy)");
- sink.process();
-
- logger.info("Running process(). Create new file.");
- sink.process();
-
- logger.info("Running process(). Create new file.");
- sink.process();
-
- logger.info("Running process(). Create new file.");
- sink.process();
-
- // shut down flume
- sink.stop();
- channel.stop();
-
- // verify that it's in HDFS and that its content is what we say it should be
- FileStatus[] statuses = fs.listStatus(outputDirPath);
- Assert.assertNotNull("No files found written to HDFS", statuses);
-
- for (FileStatus status : statuses) {
- Path filePath = status.getPath();
- logger.info("Found file on DFS: {}", filePath);
- FSDataInputStream stream = fs.open(filePath);
- BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
- String line = reader.readLine();
- logger.info("First line in file {}: {}", filePath, line);
- Assert.assertTrue(line.startsWith("yarg"));
- }
-
- Assert.assertTrue("4 or 5 files expected, found " + statuses.length,
- statuses.length == 4 || statuses.length == 5);
- System.out.println("There are " + statuses.length + " files.");
-
- if (!KEEP_DATA) {
- fs.delete(outputDirPath, true);
- }
-
- cluster.shutdown();
- cluster = null;
- }
-
- /**
- * This is a very basic test that writes one event to HDFS and reads it back.
- */
- @Ignore("This test is flakey and causes tests to fail pretty often.")
- @Test
- public void maxUnderReplicationTest() throws EventDeliveryException,
- IOException {
- Configuration conf = new Configuration();
- conf.set("dfs.replication", String.valueOf(3));
- cluster = new MiniDFSCluster(conf, 3, true, null);
- cluster.waitActive();
-
- String outputDir = "/flume/underReplicationTest";
- Path outputDirPath = new Path(outputDir);
-
- logger.info("Running test with output dir: {}", outputDir);
-
- FileSystem fs = cluster.getFileSystem();
- // ensure output directory is empty
- if (fs.exists(outputDirPath)) {
- fs.delete(outputDirPath, true);
- }
-
- String nnURL = getNameNodeURL(cluster);
- logger.info("Namenode address: {}", nnURL);
-
- Context chanCtx = new Context();
- MemoryChannel channel = new MemoryChannel();
- channel.setName("simpleHDFSTest-mem-chan");
- channel.configure(chanCtx);
- channel.start();
-
- Context sinkCtx = new Context();
- sinkCtx.put("hdfs.path", nnURL + outputDir);
- sinkCtx.put("hdfs.fileType", HDFSWriterFactory.DataStreamType);
- sinkCtx.put("hdfs.batchSize", Integer.toString(1));
-
- HDFSEventSink sink = new HDFSEventSink();
- sink.setName("simpleHDFSTest-hdfs-sink");
- sink.configure(sinkCtx);
- sink.setChannel(channel);
- sink.start();
-
- // create an event
- channel.getTransaction().begin();
- try {
- for (int i = 0; i < 50; i++) {
- channel.put(EventBuilder.withBody("yarg " + i, Charsets.UTF_8));
- }
- channel.getTransaction().commit();
- } finally {
- channel.getTransaction().close();
- }
-
- // store events to HDFS
- logger.info("Running process(). Create new file.");
- sink.process(); // create new file;
- logger.info("Running process(). Same file.");
- sink.process();
-
- // kill a datanode
- logger.info("Killing datanode #1...");
- cluster.stopDataNode(0);
-
- // there is a race here.. the client may or may not notice that the
- // datanode is dead before it next sync()s.
- // so, this next call may or may not roll a new file.
-
- logger.info("Running process(). Create new file? (racy)");
- sink.process();
-
- for (int i = 3; i < 50; i++) {
- logger.info("Running process().");
- sink.process();
- }
-
- // shut down flume
- sink.stop();
- channel.stop();
-
- // verify that it's in HDFS and that its content is what we say it should be
- FileStatus[] statuses = fs.listStatus(outputDirPath);
- Assert.assertNotNull("No files found written to HDFS", statuses);
-
- for (FileStatus status : statuses) {
- Path filePath = status.getPath();
- logger.info("Found file on DFS: {}", filePath);
- FSDataInputStream stream = fs.open(filePath);
- BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
- String line = reader.readLine();
- logger.info("First line in file {}: {}", filePath, line);
- Assert.assertTrue(line.startsWith("yarg"));
- }
-
- System.out.println("There are " + statuses.length + " files.");
- Assert.assertEquals("31 files expected, found " + statuses.length,
- 31, statuses.length);
-
- if (!KEEP_DATA) {
- fs.delete(outputDirPath, true);
- }
-
- cluster.shutdown();
- cluster = null;
- }
-
- /**
- * Tests if the lease gets released if the close() call throws IOException.
- * For more details see https://issues.apache.org/jira/browse/FLUME-3080
- */
- @Test
- public void testLeaseRecoveredIfCloseThrowsIOException() throws Exception {
- testLeaseRecoveredIfCloseFails(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- throw new IOException();
- }
- });
- }
-
- /**
- * Tests if the lease gets released if the close() call times out.
- * For more details see https://issues.apache.org/jira/browse/FLUME-3080
- */
- @Test
- public void testLeaseRecoveredIfCloseTimesOut() throws Exception {
- testLeaseRecoveredIfCloseFails(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- TimeUnit.SECONDS.sleep(30);
- return null;
- }
- });
- }
-
- private void testLeaseRecoveredIfCloseFails(final Callable<?> doThisInClose)
- throws Exception {
- cluster = new MiniDFSCluster.Builder(new Configuration()).numDataNodes(1).format(true).build();
- cluster.waitActive();
-
- String outputDir = "/flume/leaseRecovery";
- Path outputDirPath = new Path(outputDir);
-
- logger.info("Running test with output dir: {}", outputDir);
-
- FileSystem fs = cluster.getFileSystem();
- // ensure output directory is empty
- if (fs.exists(outputDirPath)) {
- fs.delete(outputDirPath, true);
- }
- String nnURL = getNameNodeURL(cluster);
-
- Context ctx = new Context();
- MemoryChannel channel = new MemoryChannel();
- channel.configure(ctx);
- channel.start();
-
- ctx.put("hdfs.path", nnURL + outputDir);
- ctx.put("hdfs.fileType", HDFSWriterFactory.DataStreamType);
- ctx.put("hdfs.batchSize", Integer.toString(1));
- ctx.put("hdfs.callTimeout", Integer.toString(1000));
- ctx.put("hdfs.retryInterval", "10"); //to speed up test
-
- HDFSWriter hdfsWriter = new HDFSDataStream() {
- @Override
- public void close() throws IOException {
- try {
- doThisInClose.call();
- } catch (Throwable e) {
- Throwables.propagateIfPossible(e, IOException.class);
- throw new RuntimeException(e);
- }
- }
- };
- hdfsWriter.configure(ctx);
-
- HDFSEventSink sink = new HDFSEventSink();
- sink.configure(ctx);
- sink.setMockFs(fs);
- sink.setMockWriter(hdfsWriter);
- sink.setChannel(channel);
- sink.start();
-
- Transaction txn = channel.getTransaction();
- txn.begin();
- try {
- channel.put(EventBuilder.withBody("test", Charsets.UTF_8));
- txn.commit();
- } finally {
- txn.close();
- }
-
- sink.process();
- sink.stop();
- channel.stop();
-
- FileStatus[] statuses = fs.listStatus(outputDirPath);
- Assert.assertEquals(1, statuses.length);
-
- String filePath = statuses[0].getPath().toUri().getPath();
-
- // -1 in case that the lease doesn't exist.
- long leaseRenewalTime = NameNodeAdapter.getLeaseRenewalTime(cluster.getNameNode(), filePath);
- // wait until the NameNode recovers the lease
- for (int i = 0; (i < 10) && (leaseRenewalTime != -1L); i++) {
- TimeUnit.SECONDS.sleep(1);
- leaseRenewalTime = NameNodeAdapter.getLeaseRenewalTime(cluster.getNameNode(), filePath);
- }
-
- // There should be no lease for the given path even if close failed as the BucketWriter
- // explicitly calls the recoverLease()
- Assert.assertEquals(-1L, leaseRenewalTime);
-
- if (!KEEP_DATA) {
- fs.delete(outputDirPath, true);
- }
-
- cluster.shutdown();
- cluster = null;
- }
-
- @AfterClass
- public static void teardownClass() {
- // restore system state, if needed
- if (oldTestBuildDataProp != null) {
- System.setProperty(TEST_BUILD_DATA_KEY, oldTestBuildDataProp);
- }
-
- if (!KEEP_DATA) {
- FileUtils.deleteQuietly(new File(DFS_DIR));
- }
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestSequenceFileSerializerFactory.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestSequenceFileSerializerFactory.java
deleted file mode 100644
index 974e857c1..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestSequenceFileSerializerFactory.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import org.apache.flume.Context;
-import org.junit.Test;
-
-import static org.junit.Assert.assertTrue;
-
-public class TestSequenceFileSerializerFactory {
-
- @Test
- public void getTextFormatter() {
- SequenceFileSerializer formatter =
- SequenceFileSerializerFactory.getSerializer("Text", new Context());
-
- assertTrue(formatter != null);
- assertTrue(formatter.getClass().getName(),
- formatter instanceof HDFSTextSerializer);
- }
-
- @Test
- public void getWritableFormatter() {
- SequenceFileSerializer formatter =
- SequenceFileSerializerFactory.getSerializer("Writable", new Context());
-
- assertTrue(formatter != null);
- assertTrue(formatter.getClass().getName(),
- formatter instanceof HDFSWritableSerializer);
- }
-
- @Test
- public void getCustomFormatter() {
- SequenceFileSerializer formatter = SequenceFileSerializerFactory.getSerializer(
- "org.apache.flume.sink.hdfs.MyCustomSerializer$Builder", new Context());
-
- assertTrue(formatter != null);
- assertTrue(formatter.getClass().getName(),
- formatter instanceof MyCustomSerializer);
- }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestUseRawLocalFileSystem.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestUseRawLocalFileSystem.java
deleted file mode 100644
index f3e7d10a7..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestUseRawLocalFileSystem.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import java.io.File;
-import org.apache.commons.io.FileUtils;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.event.EventBuilder;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.GzipCodec;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-
-public class TestUseRawLocalFileSystem {
-
- private static Logger logger =
- LoggerFactory.getLogger(TestUseRawLocalFileSystem.class);
- private Context context;
-
- private File baseDir;
- private File testFile;
- private Event event;
-
- @Before
- public void setup() throws Exception {
- baseDir = Files.createTempDir();
- testFile = new File(baseDir.getAbsoluteFile(), "test");
- context = new Context();
- event = EventBuilder.withBody("test", Charsets.UTF_8);
- }
-
- @After
- public void teardown() throws Exception {
- FileUtils.deleteQuietly(baseDir);
- }
-
- @Test
- public void testTestFile() throws Exception {
- String file = testFile.getCanonicalPath();
- HDFSDataStream stream = new HDFSDataStream();
- context.put("hdfs.useRawLocalFileSystem", "true");
- stream.configure(context);
- stream.open(file);
- stream.append(event);
- stream.sync();
- Assert.assertTrue(testFile.length() > 0);
- }
- @Test
- public void testCompressedFile() throws Exception {
- String file = testFile.getCanonicalPath();
- HDFSCompressedDataStream stream = new HDFSCompressedDataStream();
- context.put("hdfs.useRawLocalFileSystem", "true");
- stream.configure(context);
- stream.open(file, new GzipCodec(), CompressionType.RECORD);
- stream.append(event);
- stream.sync();
- Assert.assertTrue(testFile.length() > 0);
- }
- @Test
- public void testSequenceFile() throws Exception {
- String file = testFile.getCanonicalPath();
- HDFSSequenceFile stream = new HDFSSequenceFile();
- context.put("hdfs.useRawLocalFileSystem", "true");
- stream.configure(context);
- stream.open(file);
- stream.append(event);
- stream.sync();
- Assert.assertTrue(testFile.length() > 0);
- }
-
-}
\ No newline at end of file
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/resources/log4j2.xml b/flume-ng-sinks/flume-hdfs-sink/src/test/resources/log4j2.xml
deleted file mode 100644
index 91a23f8fa..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/resources/log4j2.xml
+++ /dev/null
@@ -1,33 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-<Configuration status="OFF">
- <Appenders>
- <Console name="Console" target="SYSTEM_OUT">
- <PatternLayout pattern="%d (%t) [%p - %l] %m%n" />
- </Console>
- </Appenders>
- <Loggers>
- <Logger name="org.apache.flume" level="DEBUG"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
- <Logger name="org.mortbay" level="WARN"/>
- <Root level="INFO">
- <AppenderRef ref="Console" />
- </Root>
- </Loggers>
-</Configuration>
\ No newline at end of file
diff --git a/flume-ng-sinks/flume-hive-sink/pom.xml b/flume-ng-sinks/flume-hive-sink/pom.xml
deleted file mode 100644
index 0047b9912..000000000
--- a/flume-ng-sinks/flume-hive-sink/pom.xml
+++ /dev/null
@@ -1,149 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-sinks</artifactId>
- <version>1.11.1-SNAPSHOT</version>
- </parent>
-
- <properties>
- <!-- TODO fix spotbugs/pmd violations -->
- <spotbugs.maxAllowedViolations>6</spotbugs.maxAllowedViolations>
- <pmd.maxAllowedViolations>7</pmd.maxAllowedViolations>
- <module.name>org.apache.flume.sink.hive</module.name>
- </properties>
-
- <groupId>org.apache.flume.flume-ng-sinks</groupId>
- <artifactId>flume-hive-sink</artifactId>
- <name>Flume NG Hive Sink</name>
-
- <repositories>
- <!-- Our current hive version needs a calcite version that has dependencies not available in Maven Central.
- The pom from calcite gives the HTTP version of this repo address, which gets ignored by current maven.
- The original conjars repo is dead. Use the read-only mirror.
- -->
- <repository>
- <releases>
- <enabled>true</enabled>
- <updatePolicy>always</updatePolicy>
- <checksumPolicy>warn</checksumPolicy>
- </releases>
- <id>conjars</id>
- <name>Conjars</name>
- <url>https://conjars.wensel.net/repo/</url>
- <layout>default</layout>
- </repository>
- </repositories>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.glassfish</groupId>
- <artifactId>javax.el</artifactId>
- <version>3.0.1-b12</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-sdk</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-configuration</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hive.hcatalog</groupId>
- <artifactId>hive-hcatalog-streaming</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hive.hcatalog</groupId>
- <artifactId>hive-hcatalog-core</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-cli</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>test</scope>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <scope>test</scope>
- </dependency>
-
- <!-- end temporary -->
-
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
-
-</project>
diff --git a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/Config.java b/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/Config.java
deleted file mode 100644
index b2d258227..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/Config.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hive;
-
-public class Config {
- public static final String HIVE_METASTORE = "hive.metastore";
- public static final String HIVE_DATABASE = "hive.database";
- public static final String HIVE_TABLE = "hive.table";
- public static final String HIVE_PARTITION = "hive.partition";
- public static final String HIVE_TXNS_PER_BATCH_ASK = "hive.txnsPerBatchAsk";
- public static final String BATCH_SIZE = "batchSize";
- public static final String IDLE_TIMEOUT = "idleTimeout";
- public static final String CALL_TIMEOUT = "callTimeout";
- public static final String HEART_BEAT_INTERVAL = "heartBeatInterval";
- public static final String MAX_OPEN_CONNECTIONS = "maxOpenConnections";
- public static final String USE_LOCAL_TIME_STAMP = "useLocalTimeStamp";
- public static final String TIME_ZONE = "timeZone";
- public static final String ROUND_UNIT = "roundUnit";
- public static final String ROUND = "round";
- public static final String HOUR = "hour";
- public static final String MINUTE = "minute";
- public static final String SECOND = "second";
- public static final String ROUND_VALUE = "roundValue";
- public static final String SERIALIZER = "serializer";
-}
diff --git a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveDelimitedTextSerializer.java b/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveDelimitedTextSerializer.java
deleted file mode 100644
index 59520e7d8..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveDelimitedTextSerializer.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hive;
-
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.hive.hcatalog.streaming.DelimitedInputWriter;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.apache.hive.hcatalog.streaming.RecordWriter;
-import org.apache.hive.hcatalog.streaming.StreamingException;
-import org.apache.hive.hcatalog.streaming.TransactionBatch;
-
-import java.io.IOException;
-import java.util.Collection;
-
-/** Forwards the incoming event body to Hive unmodified
- * Sets up the delimiter and the field to column mapping
- */
-public class HiveDelimitedTextSerializer implements HiveEventSerializer {
- public static final String ALIAS = "DELIMITED";
-
- public static final String defaultDelimiter = ",";
- public static final String SERIALIZER_DELIMITER = "serializer.delimiter";
- public static final String SERIALIZER_FIELDNAMES = "serializer.fieldnames";
- public static final String SERIALIZER_SERDE_SEPARATOR = "serializer.serdeSeparator";
-
- private String delimiter;
- private String[] fieldToColMapping = null;
- private Character serdeSeparator = null;
-
- @Override
- public void write(TransactionBatch txnBatch, Event e)
- throws StreamingException, IOException, InterruptedException {
- txnBatch.write(e.getBody());
- }
-
- @Override
- public void write(TransactionBatch txnBatch, Collection<byte[]> events)
- throws StreamingException, IOException, InterruptedException {
- txnBatch.write(events);
- }
-
-
- @Override
- public RecordWriter createRecordWriter(HiveEndPoint endPoint)
- throws StreamingException, IOException, ClassNotFoundException {
- if (serdeSeparator == null) {
- return new DelimitedInputWriter(fieldToColMapping, delimiter, endPoint);
- }
- return new DelimitedInputWriter(fieldToColMapping, delimiter, endPoint, null, serdeSeparator);
- }
-
- @Override
- public void configure(Context context) {
- delimiter = parseDelimiterSpec(
- context.getString(SERIALIZER_DELIMITER, defaultDelimiter) );
- String fieldNames = context.getString(SERIALIZER_FIELDNAMES);
- if (fieldNames == null) {
- throw new IllegalArgumentException("serializer.fieldnames is not specified " +
- "for serializer " + this.getClass().getName() );
- }
- String serdeSeparatorStr = context.getString(SERIALIZER_SERDE_SEPARATOR);
- this.serdeSeparator = parseSerdeSeparatorSpec(serdeSeparatorStr);
-
- // split, but preserve empty fields (-1)
- fieldToColMapping = fieldNames.trim().split(",",-1);
- }
-
- // if delimiter is a double quoted like "\t", drop quotes
- private static String parseDelimiterSpec(String delimiter) {
- if (delimiter == null) {
- return null;
- }
- if (delimiter.charAt(0) == '"' &&
- delimiter.charAt(delimiter.length() - 1) == '"') {
- return delimiter.substring(1,delimiter.length() - 1);
- }
- return delimiter;
- }
-
- // if delimiter is a single quoted character like '\t', drop quotes
- private static Character parseSerdeSeparatorSpec(String separatorStr) {
- if (separatorStr == null) {
- return null;
- }
- if (separatorStr.length() == 1) {
- return separatorStr.charAt(0);
- }
- if (separatorStr.length() == 3 &&
- separatorStr.charAt(2) == '\'' &&
- separatorStr.charAt(separatorStr.length() - 1) == '\'') {
- return separatorStr.charAt(1);
- }
-
- throw new IllegalArgumentException("serializer.serdeSeparator spec is invalid " +
- "for " + ALIAS + " serializer " );
- }
-
-}
diff --git a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveEventSerializer.java b/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveEventSerializer.java
deleted file mode 100644
index 7ed2c8276..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveEventSerializer.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hive;
-
-import org.apache.flume.Event;
-import org.apache.flume.conf.Configurable;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.apache.hive.hcatalog.streaming.RecordWriter;
-import org.apache.hive.hcatalog.streaming.StreamingException;
-import org.apache.hive.hcatalog.streaming.TransactionBatch;
-
-import java.io.IOException;
-import java.util.Collection;
-
-public interface HiveEventSerializer extends Configurable {
- public void write(TransactionBatch batch, Event e)
- throws StreamingException, IOException, InterruptedException;
-
- public void write(TransactionBatch txnBatch, Collection<byte[]> events)
- throws StreamingException, IOException, InterruptedException;
-
- RecordWriter createRecordWriter(HiveEndPoint endPoint)
- throws StreamingException, IOException, ClassNotFoundException;
-
-}
diff --git a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveJsonSerializer.java b/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveJsonSerializer.java
deleted file mode 100644
index 0311a5b7d..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveJsonSerializer.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hive;
-
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.apache.hive.hcatalog.streaming.RecordWriter;
-import org.apache.hive.hcatalog.streaming.StreamingException;
-import org.apache.hive.hcatalog.streaming.StrictJsonWriter;
-import org.apache.hive.hcatalog.streaming.TransactionBatch;
-
-import java.io.IOException;
-import java.util.Collection;
-
-/** Forwards the incoming event body to Hive unmodified
- * Sets up the delimiter and the field to column mapping
- */
-
-public class HiveJsonSerializer implements HiveEventSerializer {
- public static final String ALIAS = "JSON";
-
- @Override
- public void write(TransactionBatch txnBatch, Event e)
- throws StreamingException, IOException, InterruptedException {
- txnBatch.write(e.getBody());
- }
-
- @Override
- public void write(TransactionBatch txnBatch, Collection<byte[]> events)
- throws StreamingException, IOException, InterruptedException {
- txnBatch.write(events);
- }
-
- @Override
- public RecordWriter createRecordWriter(HiveEndPoint endPoint)
- throws StreamingException, IOException, ClassNotFoundException {
- return new StrictJsonWriter(endPoint);
- }
-
- @Override
- public void configure(Context context) {
- return;
- }
-
-}
diff --git a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveSink.java b/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveSink.java
deleted file mode 100644
index ec1fe621c..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveSink.java
+++ /dev/null
@@ -1,528 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hive;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.flume.Channel;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.Transaction;
-import org.apache.flume.conf.BatchSizeSupported;
-import org.apache.flume.conf.Configurable;
-import org.apache.flume.formatter.output.BucketPath;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.flume.sink.AbstractSink;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Calendar;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TimeZone;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class HiveSink extends AbstractSink implements Configurable, BatchSizeSupported {
-
- private static final Logger LOG = LoggerFactory.getLogger(HiveSink.class);
-
- private static final int DEFAULT_MAXOPENCONNECTIONS = 500;
- private static final int DEFAULT_TXNSPERBATCH = 100;
- private static final int DEFAULT_BATCHSIZE = 15000;
- private static final int DEFAULT_CALLTIMEOUT = 10000;
- private static final int DEFAULT_IDLETIMEOUT = 0;
- private static final int DEFAULT_HEARTBEATINTERVAL = 240; // seconds
-
- private Map<HiveEndPoint, HiveWriter> allWriters;
-
- private SinkCounter sinkCounter;
- private volatile int idleTimeout;
- private String metaStoreUri;
- private String proxyUser;
- private String database;
- private String table;
- private List<String> partitionVals;
- private Integer txnsPerBatchAsk;
- private Integer batchSize;
- private Integer maxOpenConnections;
- private boolean autoCreatePartitions;
- private String serializerType;
- private HiveEventSerializer serializer;
-
- /**
- * Default timeout for blocking I/O calls in HiveWriter
- */
- private Integer callTimeout;
- private Integer heartBeatInterval;
-
- private ExecutorService callTimeoutPool;
-
- private boolean useLocalTime;
- private TimeZone timeZone;
- private boolean needRounding;
- private int roundUnit;
- private Integer roundValue;
-
- private Timer heartBeatTimer = new Timer();
- private AtomicBoolean timeToSendHeartBeat = new AtomicBoolean(false);
-
- @VisibleForTesting
- Map<HiveEndPoint, HiveWriter> getAllWriters() {
- return allWriters;
- }
-
- // read configuration and setup thresholds
- @Override
- public void configure(Context context) {
-
- metaStoreUri = context.getString(Config.HIVE_METASTORE);
- if (metaStoreUri == null) {
- throw new IllegalArgumentException(Config.HIVE_METASTORE + " config setting is not " +
- "specified for sink " + getName());
- }
- if (metaStoreUri.equalsIgnoreCase("null")) { // for testing support
- metaStoreUri = null;
- }
- proxyUser = null; // context.getString("hive.proxyUser"); not supported by hive api yet
- database = context.getString(Config.HIVE_DATABASE);
- if (database == null) {
- throw new IllegalArgumentException(Config.HIVE_DATABASE + " config setting is not " +
- "specified for sink " + getName());
- }
- table = context.getString(Config.HIVE_TABLE);
- if (table == null) {
- throw new IllegalArgumentException(Config.HIVE_TABLE + " config setting is not " +
- "specified for sink " + getName());
- }
-
- String partitions = context.getString(Config.HIVE_PARTITION);
- if (partitions != null) {
- partitionVals = Arrays.asList(partitions.split(","));
- }
-
-
- txnsPerBatchAsk = context.getInteger(Config.HIVE_TXNS_PER_BATCH_ASK, DEFAULT_TXNSPERBATCH);
- if (txnsPerBatchAsk < 0) {
- LOG.warn(getName() + ". hive.txnsPerBatchAsk must be positive number. Defaulting to "
- + DEFAULT_TXNSPERBATCH);
- txnsPerBatchAsk = DEFAULT_TXNSPERBATCH;
- }
- batchSize = context.getInteger(Config.BATCH_SIZE, DEFAULT_BATCHSIZE);
- if (batchSize < 0) {
- LOG.warn(getName() + ". batchSize must be positive number. Defaulting to "
- + DEFAULT_BATCHSIZE);
- batchSize = DEFAULT_BATCHSIZE;
- }
- idleTimeout = context.getInteger(Config.IDLE_TIMEOUT, DEFAULT_IDLETIMEOUT);
- if (idleTimeout < 0) {
- LOG.warn(getName() + ". idleTimeout must be positive number. Defaulting to "
- + DEFAULT_IDLETIMEOUT);
- idleTimeout = DEFAULT_IDLETIMEOUT;
- }
- callTimeout = context.getInteger(Config.CALL_TIMEOUT, DEFAULT_CALLTIMEOUT);
- if (callTimeout < 0) {
- LOG.warn(getName() + ". callTimeout must be positive number. Defaulting to "
- + DEFAULT_CALLTIMEOUT);
- callTimeout = DEFAULT_CALLTIMEOUT;
- }
-
- heartBeatInterval = context.getInteger(Config.HEART_BEAT_INTERVAL, DEFAULT_HEARTBEATINTERVAL);
- if (heartBeatInterval < 0) {
- LOG.warn(getName() + ". heartBeatInterval must be positive number. Defaulting to "
- + DEFAULT_HEARTBEATINTERVAL);
- heartBeatInterval = DEFAULT_HEARTBEATINTERVAL;
- }
- maxOpenConnections = context.getInteger(Config.MAX_OPEN_CONNECTIONS,
- DEFAULT_MAXOPENCONNECTIONS);
- autoCreatePartitions = context.getBoolean("autoCreatePartitions", true);
-
- // Timestamp processing
- useLocalTime = context.getBoolean(Config.USE_LOCAL_TIME_STAMP, false);
-
- String tzName = context.getString(Config.TIME_ZONE);
- timeZone = (tzName == null) ? null : TimeZone.getTimeZone(tzName);
- needRounding = context.getBoolean(Config.ROUND, false);
-
- String unit = context.getString(Config.ROUND_UNIT, Config.MINUTE);
- if (unit.equalsIgnoreCase(Config.HOUR)) {
- this.roundUnit = Calendar.HOUR_OF_DAY;
- } else if (unit.equalsIgnoreCase(Config.MINUTE)) {
- this.roundUnit = Calendar.MINUTE;
- } else if (unit.equalsIgnoreCase(Config.SECOND)) {
- this.roundUnit = Calendar.SECOND;
- } else {
- LOG.warn(getName() + ". Rounding unit is not valid, please set one of " +
- "minute, hour or second. Rounding will be disabled");
- needRounding = false;
- }
- this.roundValue = context.getInteger(Config.ROUND_VALUE, 1);
- if (roundUnit == Calendar.SECOND || roundUnit == Calendar.MINUTE) {
- Preconditions.checkArgument(roundValue > 0 && roundValue <= 60,
- "Round value must be > 0 and <= 60");
- } else if (roundUnit == Calendar.HOUR_OF_DAY) {
- Preconditions.checkArgument(roundValue > 0 && roundValue <= 24,
- "Round value must be > 0 and <= 24");
- }
-
- // Serializer
- serializerType = context.getString(Config.SERIALIZER, "");
- if (serializerType.isEmpty()) {
- throw new IllegalArgumentException("serializer config setting is not " +
- "specified for sink " + getName());
- }
-
- serializer = createSerializer(serializerType);
- serializer.configure(context);
-
- Preconditions.checkArgument(batchSize > 0, "batchSize must be greater than 0");
-
- if (sinkCounter == null) {
- sinkCounter = new SinkCounter(getName());
- }
- }
-
- @VisibleForTesting
- protected SinkCounter getCounter() {
- return sinkCounter;
- }
- private HiveEventSerializer createSerializer(String serializerName) {
- if (serializerName.compareToIgnoreCase(HiveDelimitedTextSerializer.ALIAS) == 0 ||
- serializerName.compareTo(HiveDelimitedTextSerializer.class.getName()) == 0) {
- return new HiveDelimitedTextSerializer();
- } else if (serializerName.compareToIgnoreCase(HiveJsonSerializer.ALIAS) == 0 ||
- serializerName.compareTo(HiveJsonSerializer.class.getName()) == 0) {
- return new HiveJsonSerializer();
- }
-
- try {
- return (HiveEventSerializer) Class.forName(serializerName).newInstance();
- } catch (Exception e) {
- throw new IllegalArgumentException("Unable to instantiate serializer: " + serializerName
- + " on sink: " + getName(), e);
- }
- }
-
-
- /**
- * Pull events out of channel, find corresponding HiveWriter and write to it.
- * Take at most batchSize events per Transaction. <br/>
- * This method is not thread safe.
- */
- public Status process() throws EventDeliveryException {
- // writers used in this Txn
-
- Channel channel = getChannel();
- Transaction transaction = channel.getTransaction();
- transaction.begin();
- boolean success = false;
- try {
- // 1 Enable Heart Beats
- if (timeToSendHeartBeat.compareAndSet(true, false)) {
- enableHeartBeatOnAllWriters();
- }
-
- // 2 Drain Batch
- int txnEventCount = drainOneBatch(channel);
- transaction.commit();
- success = true;
-
- // 3 Update Counters
- if (txnEventCount < 1) {
- return Status.BACKOFF;
- } else {
- return Status.READY;
- }
- } catch (InterruptedException err) {
- LOG.warn(getName() + ": Thread was interrupted.", err);
- return Status.BACKOFF;
- } catch (Exception e) {
- sinkCounter.incrementEventWriteOrChannelFail(e);
- throw new EventDeliveryException(e);
- } finally {
- if (!success) {
- transaction.rollback();
- }
- transaction.close();
- }
- }
-
- // Drains one batch of events from Channel into Hive
- private int drainOneBatch(Channel channel)
- throws HiveWriter.Failure, InterruptedException {
- int txnEventCount = 0;
- try {
- Map<HiveEndPoint,HiveWriter> activeWriters = Maps.newHashMap();
- for (; txnEventCount < batchSize; ++txnEventCount) {
- // 0) Read event from Channel
- Event event = channel.take();
- if (event == null) {
- break;
- }
-
- //1) Create end point by substituting place holders
- HiveEndPoint endPoint = makeEndPoint(metaStoreUri, database, table,
- partitionVals, event.getHeaders(), timeZone,
- needRounding, roundUnit, roundValue, useLocalTime);
-
- //2) Create or reuse Writer
- HiveWriter writer = getOrCreateWriter(activeWriters, endPoint);
-
- //3) Write
- LOG.debug("{} : Writing event to {}", getName(), endPoint);
- writer.write(event);
-
- } // for
-
- //4) Update counters
- if (txnEventCount == 0) {
- sinkCounter.incrementBatchEmptyCount();
- } else if (txnEventCount == batchSize) {
- sinkCounter.incrementBatchCompleteCount();
- } else {
- sinkCounter.incrementBatchUnderflowCount();
- }
- sinkCounter.addToEventDrainAttemptCount(txnEventCount);
-
-
- // 5) Flush all Writers
- for (HiveWriter writer : activeWriters.values()) {
- writer.flush(true);
- }
-
- sinkCounter.addToEventDrainSuccessCount(txnEventCount);
- return txnEventCount;
- } catch (HiveWriter.Failure e) {
- // in case of error we close all TxnBatches to start clean next time
- LOG.warn(getName() + " : " + e.getMessage(), e);
- abortAllWriters();
- closeAllWriters();
- throw e;
- }
- }
-
- private void enableHeartBeatOnAllWriters() {
- for (HiveWriter writer : allWriters.values()) {
- writer.setHearbeatNeeded();
- }
- }
-
- private HiveWriter getOrCreateWriter(Map<HiveEndPoint, HiveWriter> activeWriters,
- HiveEndPoint endPoint)
- throws HiveWriter.ConnectException, InterruptedException {
- try {
- HiveWriter writer = allWriters.get( endPoint );
- if (writer == null) {
- LOG.info(getName() + ": Creating Writer to Hive end point : " + endPoint);
- writer = new HiveWriter(endPoint, txnsPerBatchAsk, autoCreatePartitions,
- callTimeout, callTimeoutPool, proxyUser, serializer, sinkCounter);
-
- sinkCounter.incrementConnectionCreatedCount();
- if (allWriters.size() > maxOpenConnections) {
- int retired = closeIdleWriters();
- if (retired == 0) {
- closeEldestWriter();
- }
- }
- allWriters.put(endPoint, writer);
- activeWriters.put(endPoint, writer);
- } else {
- if (activeWriters.get(endPoint) == null) {
- activeWriters.put(endPoint,writer);
- }
- }
- return writer;
- } catch (HiveWriter.ConnectException e) {
- sinkCounter.incrementConnectionFailedCount();
- throw e;
- }
-
- }
-
- private HiveEndPoint makeEndPoint(String metaStoreUri, String database, String table,
- List<String> partVals, Map<String, String> headers,
- TimeZone timeZone, boolean needRounding,
- int roundUnit, Integer roundValue,
- boolean useLocalTime) {
- if (partVals == null) {
- return new HiveEndPoint(metaStoreUri, database, table, null);
- }
-
- ArrayList<String> realPartVals = Lists.newArrayList();
- for (String partVal : partVals) {
- realPartVals.add(BucketPath.escapeString(partVal, headers, timeZone,
- needRounding, roundUnit, roundValue, useLocalTime));
- }
- return new HiveEndPoint(metaStoreUri, database, table, realPartVals);
- }
-
- /**
- * Locate writer that has not been used for longest time and retire it
- */
- private void closeEldestWriter() throws InterruptedException {
- long oldestTimeStamp = System.currentTimeMillis();
- HiveEndPoint eldest = null;
- for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
- if (entry.getValue().getLastUsed() < oldestTimeStamp) {
- eldest = entry.getKey();
- oldestTimeStamp = entry.getValue().getLastUsed();
- }
- }
-
- try {
- sinkCounter.incrementConnectionCreatedCount();
- LOG.info(getName() + ": Closing least used Writer to Hive EndPoint : " + eldest);
- allWriters.remove(eldest).close();
- } catch (InterruptedException e) {
- LOG.warn(getName() + ": Interrupted when attempting to close writer for end point: "
- + eldest, e);
- throw e;
- }
- }
-
- /**
- * Locate all writers past idle timeout and retire them
- * @return number of writers retired
- */
- private int closeIdleWriters() throws InterruptedException {
- int count = 0;
- long now = System.currentTimeMillis();
- ArrayList<HiveEndPoint> retirees = Lists.newArrayList();
-
- //1) Find retirement candidates
- for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
- if (now - entry.getValue().getLastUsed() > idleTimeout) {
- ++count;
- retirees.add(entry.getKey());
- }
- }
- //2) Retire them
- for (HiveEndPoint ep : retirees) {
- sinkCounter.incrementConnectionClosedCount();
- LOG.info(getName() + ": Closing idle Writer to Hive end point : {}", ep);
- allWriters.remove(ep).close();
- }
- return count;
- }
-
- /**
- * Closes all writers and remove them from cache
- * @return number of writers retired
- */
- private void closeAllWriters() throws InterruptedException {
- //1) Retire writers
- for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
- entry.getValue().close();
- }
-
- //2) Clear cache
- allWriters.clear();
- }
-
- /**
- * Abort current Txn on all writers
- * @return number of writers retired
- */
- private void abortAllWriters() throws InterruptedException {
- for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
- entry.getValue().abort();
- }
- }
-
- @Override
- public void stop() {
- // do not constrain close() calls with a timeout
- for (Entry<HiveEndPoint, HiveWriter> entry : allWriters.entrySet()) {
- try {
- HiveWriter w = entry.getValue();
- w.close();
- } catch (InterruptedException ex) {
- Thread.currentThread().interrupt();
- }
- }
-
- // shut down all thread pools
- callTimeoutPool.shutdown();
- try {
- while (callTimeoutPool.isTerminated() == false) {
- callTimeoutPool.awaitTermination(
- Math.max(DEFAULT_CALLTIMEOUT, callTimeout), TimeUnit.MILLISECONDS);
- }
- } catch (InterruptedException ex) {
- LOG.warn(getName() + ":Shutdown interrupted on " + callTimeoutPool, ex);
- }
-
- callTimeoutPool = null;
- allWriters.clear();
- allWriters = null;
- sinkCounter.stop();
- super.stop();
- LOG.info("Hive Sink {} stopped", getName() );
- }
-
- @Override
- public void start() {
- String timeoutName = "hive-" + getName() + "-call-runner-%d";
- // call timeout pool needs only 1 thd as sink is effectively single threaded
- callTimeoutPool = Executors.newFixedThreadPool(1,
- new ThreadFactoryBuilder().setNameFormat(timeoutName).build());
-
- this.allWriters = Maps.newHashMap();
- sinkCounter.start();
- super.start();
- setupHeartBeatTimer();
- LOG.info(getName() + ": Hive Sink {} started", getName() );
- }
-
- private void setupHeartBeatTimer() {
- if (heartBeatInterval > 0) {
- heartBeatTimer.schedule(new TimerTask() {
- @Override
- public void run() {
- timeToSendHeartBeat.set(true);
- setupHeartBeatTimer();
- }
- }, heartBeatInterval * 1000);
- }
- }
-
- @Override
- public long getBatchSize() {
- return batchSize;
- }
-
- @Override
- public String toString() {
- return "{ Sink type:" + getClass().getSimpleName() + ", name:" + getName() +
- " }";
- }
-
-}
diff --git a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveWriter.java b/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveWriter.java
deleted file mode 100644
index 7106696a1..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveWriter.java
+++ /dev/null
@@ -1,513 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hive;
-
-import org.apache.flume.Event;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.apache.hive.hcatalog.streaming.RecordWriter;
-import org.apache.hive.hcatalog.streaming.SerializationError;
-import org.apache.hive.hcatalog.streaming.StreamingConnection;
-import org.apache.hive.hcatalog.streaming.StreamingException;
-import org.apache.hive.hcatalog.streaming.StreamingIOFailure;
-import org.apache.hive.hcatalog.streaming.TransactionBatch;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * Internal API intended for HiveSink use.
- */
-class HiveWriter {
-
- private static final Logger LOG = LoggerFactory.getLogger(HiveWriter.class);
-
- private final HiveEndPoint endPoint;
- private HiveEventSerializer serializer;
- private final StreamingConnection connection;
- private final int txnsPerBatch;
- private final RecordWriter recordWriter;
- private TransactionBatch txnBatch;
-
- private final ExecutorService callTimeoutPool;
-
- private final long callTimeout;
-
- private long lastUsed; // time of last flush on this writer
-
- private SinkCounter sinkCounter;
- private int batchCounter;
- private long eventCounter;
- private long processSize;
-
- protected boolean closed; // flag indicating HiveWriter was closed
- private boolean autoCreatePartitions;
-
- private boolean hearbeatNeeded = false;
-
- private final int writeBatchSz = 1000;
- private ArrayList<Event> batch = new ArrayList<Event>(writeBatchSz);
-
- HiveWriter(HiveEndPoint endPoint, int txnsPerBatch,
- boolean autoCreatePartitions, long callTimeout,
- ExecutorService callTimeoutPool, String hiveUser,
- HiveEventSerializer serializer, SinkCounter sinkCounter)
- throws ConnectException, InterruptedException {
- try {
- this.autoCreatePartitions = autoCreatePartitions;
- this.sinkCounter = sinkCounter;
- this.callTimeout = callTimeout;
- this.callTimeoutPool = callTimeoutPool;
- this.endPoint = endPoint;
- this.connection = newConnection(hiveUser);
- this.txnsPerBatch = txnsPerBatch;
- this.serializer = serializer;
- this.recordWriter = serializer.createRecordWriter(endPoint);
- this.txnBatch = nextTxnBatch(recordWriter);
- this.txnBatch.beginNextTransaction();
- this.closed = false;
- this.lastUsed = System.currentTimeMillis();
- } catch (InterruptedException e) {
- throw e;
- } catch (RuntimeException e) {
- throw e;
- } catch (Exception e) {
- throw new ConnectException(endPoint, e);
- }
- }
-
- @Override
- public String toString() {
- return endPoint.toString();
- }
-
- /**
- * Clear the class counters
- */
- private void resetCounters() {
- eventCounter = 0;
- processSize = 0;
- batchCounter = 0;
- }
-
- void setHearbeatNeeded() {
- hearbeatNeeded = true;
- }
-
- public int getRemainingTxns() {
- return txnBatch.remainingTransactions();
- }
-
-
- /**
- * Write data, update stats
- * @param event
- * @throws WriteException - other streaming io error
- * @throws InterruptedException
- */
- public synchronized void write(final Event event)
- throws WriteException, InterruptedException {
- if (closed) {
- throw new IllegalStateException("Writer closed. Cannot write to : " + endPoint);
- }
-
- batch.add(event);
- if (batch.size() == writeBatchSz) {
- // write the event
- writeEventBatchToSerializer();
- }
-
- // Update Statistics
- processSize += event.getBody().length;
- eventCounter++;
- }
-
- private void writeEventBatchToSerializer()
- throws InterruptedException, WriteException {
- try {
- timedCall(new CallRunner1<Void>() {
- @Override
- public Void call() throws InterruptedException, StreamingException {
- try {
- for (Event event : batch) {
- try {
- serializer.write(txnBatch, event);
- } catch (SerializationError err) {
- LOG.info("Parse failed : {} : {}", err.getMessage(), new String(event.getBody()));
- }
- }
- return null;
- } catch (IOException e) {
- throw new StreamingIOFailure(e.getMessage(), e);
- }
- }
- });
- batch.clear();
- } catch (StreamingException e) {
- throw new WriteException(endPoint, txnBatch.getCurrentTxnId(), e);
- } catch (TimeoutException e) {
- throw new WriteException(endPoint, txnBatch.getCurrentTxnId(), e);
- }
- }
-
- /**
- * Commits the current Txn.
- * If 'rollToNext' is true, will switch to next Txn in batch or to a
- * new TxnBatch if current Txn batch is exhausted
- */
- public void flush(boolean rollToNext)
- throws CommitException, TxnBatchException, TxnFailure, InterruptedException,
- WriteException {
- if (!batch.isEmpty()) {
- writeEventBatchToSerializer();
- batch.clear();
- }
-
- //0 Heart beat on TxnBatch
- if (hearbeatNeeded) {
- hearbeatNeeded = false;
- heartBeat();
- }
- lastUsed = System.currentTimeMillis();
-
- try {
- //1 commit txn & close batch if needed
- commitTxn();
- if (txnBatch.remainingTransactions() == 0) {
- closeTxnBatch();
- txnBatch = null;
- if (rollToNext) {
- txnBatch = nextTxnBatch(recordWriter);
- }
- }
-
- //2 roll to next Txn
- if (rollToNext) {
- LOG.debug("Switching to next Txn for {}", endPoint);
- txnBatch.beginNextTransaction(); // does not block
- }
- } catch (StreamingException e) {
- throw new TxnFailure(txnBatch, e);
- }
- }
-
- /**
- * Aborts the current Txn
- * @throws InterruptedException
- */
- public void abort() throws InterruptedException {
- batch.clear();
- abortTxn();
- }
-
- /** Queues up a heartbeat request on the current and remaining txns using the
- * heartbeatThdPool and returns immediately
- */
- public void heartBeat() throws InterruptedException {
- // 1) schedule the heartbeat on one thread in pool
- try {
- timedCall(new CallRunner1<Void>() {
- @Override
- public Void call() throws StreamingException {
- LOG.info("Sending heartbeat on batch " + txnBatch);
- txnBatch.heartbeat();
- return null;
- }
- });
- } catch (InterruptedException e) {
- throw e;
- } catch (Exception e) {
- LOG.warn("Unable to send heartbeat on Txn Batch " + txnBatch, e);
- // Suppressing exceptions as we don't care for errors on heartbeats
- }
- }
-
- /**
- * Close the Transaction Batch and connection
- * @throws IOException
- * @throws InterruptedException
- */
- public void close() throws InterruptedException {
- batch.clear();
- abortRemainingTxns();
- closeTxnBatch();
- closeConnection();
- closed = true;
- }
-
-
- private void abortRemainingTxns() throws InterruptedException {
- try {
- if (!isClosed(txnBatch.getCurrentTransactionState())) {
- abortCurrTxnHelper();
- }
-
- // recursively abort remaining txns
- if (txnBatch.remainingTransactions() > 0) {
- timedCall(
- new CallRunner1<Void>() {
- @Override
- public Void call() throws StreamingException, InterruptedException {
- txnBatch.beginNextTransaction();
- return null;
- }
- });
- abortRemainingTxns();
- }
- } catch (StreamingException e) {
- LOG.warn("Error when aborting remaining transactions in batch " + txnBatch, e);
- return;
- } catch (TimeoutException e) {
- LOG.warn("Timed out when aborting remaining transactions in batch " + txnBatch, e);
- return;
- }
- }
-
- private void abortCurrTxnHelper() throws TimeoutException, InterruptedException {
- try {
- timedCall(
- new CallRunner1<Void>() {
- @Override
- public Void call() throws StreamingException, InterruptedException {
- txnBatch.abort();
- LOG.info("Aborted txn " + txnBatch.getCurrentTxnId());
- return null;
- }
- }
- );
- } catch (StreamingException e) {
- LOG.warn("Unable to abort transaction " + txnBatch.getCurrentTxnId(), e);
- // continue to attempt to abort other txns in the batch
- }
- }
-
- private boolean isClosed(TransactionBatch.TxnState txnState) {
- if (txnState == TransactionBatch.TxnState.COMMITTED) {
- return true;
- }
- if (txnState == TransactionBatch.TxnState.ABORTED) {
- return true;
- }
- return false;
- }
-
- public void closeConnection() throws InterruptedException {
- LOG.info("Closing connection to EndPoint : {}", endPoint);
- try {
- timedCall(new CallRunner1<Void>() {
- @Override
- public Void call() {
- connection.close(); // could block
- return null;
- }
- });
- sinkCounter.incrementConnectionClosedCount();
- } catch (Exception e) {
- LOG.warn("Error closing connection to EndPoint : " + endPoint, e);
- // Suppressing exceptions as we don't care for errors on connection close
- }
- }
-
- private void commitTxn() throws CommitException, InterruptedException {
- if (LOG.isInfoEnabled()) {
- LOG.info("Committing Txn " + txnBatch.getCurrentTxnId() + " on EndPoint: " + endPoint);
- }
- try {
- timedCall(new CallRunner1<Void>() {
- @Override
- public Void call() throws StreamingException, InterruptedException {
- txnBatch.commit(); // could block
- return null;
- }
- });
- } catch (Exception e) {
- throw new CommitException(endPoint, txnBatch.getCurrentTxnId(), e);
- }
- }
-
- private void abortTxn() throws InterruptedException {
- LOG.info("Aborting Txn id {} on End Point {}", txnBatch.getCurrentTxnId(), endPoint);
- try {
- timedCall(new CallRunner1<Void>() {
- @Override
- public Void call() throws StreamingException, InterruptedException {
- txnBatch.abort(); // could block
- return null;
- }
- });
- } catch (InterruptedException e) {
- throw e;
- } catch (TimeoutException e) {
- LOG.warn("Timeout while aborting Txn " + txnBatch.getCurrentTxnId() +
- " on EndPoint: " + endPoint, e);
- } catch (Exception e) {
- LOG.warn("Error aborting Txn " + txnBatch.getCurrentTxnId() + " on EndPoint: " + endPoint, e);
- // Suppressing exceptions as we don't care for errors on abort
- }
- }
-
- private StreamingConnection newConnection(final String proxyUser)
- throws InterruptedException, ConnectException {
- try {
- return timedCall(new CallRunner1<StreamingConnection>() {
- @Override
- public StreamingConnection call() throws InterruptedException, StreamingException {
- return endPoint.newConnection(autoCreatePartitions); // could block
- }
- });
- } catch (Exception e) {
- throw new ConnectException(endPoint, e);
- }
- }
-
- private TransactionBatch nextTxnBatch(final RecordWriter recordWriter)
- throws InterruptedException, TxnBatchException {
- LOG.debug("Fetching new Txn Batch for {}", endPoint);
- TransactionBatch batch = null;
- try {
- batch = timedCall(new CallRunner1<TransactionBatch>() {
- @Override
- public TransactionBatch call() throws InterruptedException, StreamingException {
- return connection.fetchTransactionBatch(txnsPerBatch, recordWriter); // could block
- }
- });
- LOG.info("Acquired Transaction batch {}", batch);
- } catch (Exception e) {
- throw new TxnBatchException(endPoint, e);
- }
- return batch;
- }
-
- private void closeTxnBatch() throws InterruptedException {
- try {
- LOG.info("Closing Txn Batch {}.", txnBatch);
- timedCall(new CallRunner1<Void>() {
- @Override
- public Void call() throws InterruptedException, StreamingException {
- txnBatch.close(); // could block
- return null;
- }
- });
- } catch (InterruptedException e) {
- throw e;
- } catch (Exception e) {
- LOG.warn("Error closing Txn Batch " + txnBatch, e);
- // Suppressing exceptions as we don't care for errors on batch close
- }
- }
-
- private <T> T timedCall(final CallRunner1<T> callRunner)
- throws TimeoutException, InterruptedException, StreamingException {
- Future<T> future = callTimeoutPool.submit(new Callable<T>() {
- @Override
- public T call() throws StreamingException, InterruptedException, Failure {
- return callRunner.call();
- }
- });
-
- try {
- if (callTimeout > 0) {
- return future.get(callTimeout, TimeUnit.MILLISECONDS);
- } else {
- return future.get();
- }
- } catch (TimeoutException eT) {
- future.cancel(true);
- sinkCounter.incrementConnectionFailedCount();
- throw eT;
- } catch (ExecutionException e1) {
- sinkCounter.incrementConnectionFailedCount();
- Throwable cause = e1.getCause();
- if (cause instanceof IOException) {
- throw new StreamingException("I/O Failure", (IOException) cause);
- } else if (cause instanceof StreamingException) {
- throw (StreamingException) cause;
- } else if (cause instanceof TimeoutException) {
- throw new StreamingException("Operation Timed Out.", (TimeoutException) cause);
- } else if (cause instanceof RuntimeException) {
- throw (RuntimeException) cause;
- } else if (cause instanceof InterruptedException) {
- throw (InterruptedException) cause;
- }
- throw new StreamingException(e1.getMessage(), e1);
- }
- }
-
- long getLastUsed() {
- return lastUsed;
- }
-
- /**
- * Simple interface whose <tt>call</tt> method is called by
- * {#callWithTimeout} in a new thread inside a
- * {@linkplain java.security.PrivilegedExceptionAction#run()} call.
- * @param <T>
- */
- private interface CallRunner<T> {
- T call() throws Exception;
- }
-
- private interface CallRunner1<T> {
- T call() throws StreamingException, InterruptedException, Failure;
- }
-
- public static class Failure extends Exception {
- public Failure(String msg, Throwable cause) {
- super(msg, cause);
- }
- }
-
- public static class WriteException extends Failure {
- public WriteException(HiveEndPoint endPoint, Long currentTxnId, Throwable cause) {
- super("Failed writing to : " + endPoint + ". TxnID : " + currentTxnId, cause);
- }
- }
-
- public static class CommitException extends Failure {
- public CommitException(HiveEndPoint endPoint, Long txnID, Throwable cause) {
- super("Commit of Txn " + txnID + " failed on EndPoint: " + endPoint, cause);
- }
- }
-
- public static class ConnectException extends Failure {
- public ConnectException(HiveEndPoint ep, Throwable cause) {
- super("Failed connecting to EndPoint " + ep, cause);
- }
- }
-
- public static class TxnBatchException extends Failure {
- public TxnBatchException(HiveEndPoint ep, Throwable cause) {
- super("Failed acquiring Transaction Batch from EndPoint: " + ep, cause);
- }
- }
-
- private class TxnFailure extends Failure {
- public TxnFailure(TransactionBatch txnBatch, Throwable cause) {
- super("Failed switching to next Txn in TxnBatch " + txnBatch, cause);
- }
- }
-}
diff --git a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveSink.java b/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveSink.java
deleted file mode 100644
index d07ef4bec..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveSink.java
+++ /dev/null
@@ -1,413 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.flume.sink.hive;
-
-import com.google.common.collect.Lists;
-import junit.framework.Assert;
-import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.Transaction;
-import org.apache.flume.channel.BasicTransactionSemantics;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.conf.Configurables;
-import org.apache.flume.event.SimpleEvent;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.mockito.Mockito;
-import org.mockito.internal.util.reflection.Whitebox;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.List;
-import java.util.UUID;
-
-public class TestHiveSink {
- // 1) partitioned table
- static final String dbName = "testing";
- static final String tblName = "alerts";
-
- public static final String PART1_NAME = "continent";
- public static final String PART2_NAME = "country";
- public static final String[] partNames = { PART1_NAME, PART2_NAME };
-
- private static final String COL1 = "id";
- private static final String COL2 = "msg";
- final String[] colNames = {COL1,COL2};
- private String[] colTypes = { "int", "string" };
-
- private static final String PART1_VALUE = "Asia";
- private static final String PART2_VALUE = "India";
- private final ArrayList<String> partitionVals;
-
- // 2) un-partitioned table
- static final String dbName2 = "testing2";
- static final String tblName2 = "alerts2";
- final String[] colNames2 = {COL1,COL2};
- private String[] colTypes2 = { "int", "string" };
-
- HiveSink sink = new HiveSink();
-
- private final HiveConf conf;
-
- private final Driver driver;
-
- final String metaStoreURI;
-
- @Rule
- public TemporaryFolder dbFolder = new TemporaryFolder();
-
- private static final Logger LOG = LoggerFactory.getLogger(HiveSink.class);
-
- public TestHiveSink() throws Exception {
- partitionVals = new ArrayList<String>(2);
- partitionVals.add(PART1_VALUE);
- partitionVals.add(PART2_VALUE);
-
- metaStoreURI = "null";
-
- conf = new HiveConf(this.getClass());
- TestUtil.setConfValues(conf);
-
- // 1) prepare hive
- TxnDbUtil.cleanDb(conf);
- TxnDbUtil.prepDb(conf);
-
- // 2) Setup Hive client
- SessionState.start(new CliSessionState(conf));
- driver = new Driver(conf);
-
- }
-
-
- @Before
- public void setUp() throws Exception {
- TestUtil.dropDB(conf, dbName);
-
- sink = new HiveSink();
- sink.setName("HiveSink-" + UUID.randomUUID().toString());
-
- String dbLocation = dbFolder.newFolder(dbName).getCanonicalPath() + ".db";
- dbLocation = dbLocation.replaceAll("\\\\","/"); // for windows paths
- TestUtil.createDbAndTable(driver, dbName, tblName, partitionVals, colNames,
- colTypes, partNames, dbLocation);
- }
-
- @After
- public void tearDown() throws MetaException, HiveException {
- TestUtil.dropDB(conf, dbName);
- }
-
- public void testSingleWriter(boolean partitioned, String dbName, String tblName,
- Channel pChannel) throws Exception {
- int totalRecords = 4;
- int batchSize = 2;
- int batchCount = totalRecords / batchSize;
-
- Context context = new Context();
- context.put("hive.metastore", metaStoreURI);
- context.put("hive.database",dbName);
- context.put("hive.table",tblName);
- if (partitioned) {
- context.put("hive.partition", PART1_VALUE + "," + PART2_VALUE);
- }
- context.put("autoCreatePartitions","false");
- context.put("batchSize","" + batchSize);
- context.put("serializer", HiveDelimitedTextSerializer.ALIAS);
- context.put("serializer.fieldnames", COL1 + ",," + COL2 + ",");
- context.put("heartBeatInterval", "0");
-
- Channel channel = startSink(sink, context, pChannel);
-
- List<String> bodies = Lists.newArrayList();
-
- // push the events in two batches
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (int j = 1; j <= totalRecords; j++) {
- Event event = new SimpleEvent();
- String body = j + ",blah,This is a log message,other stuff";
- event.setBody(body.getBytes());
- bodies.add(body);
- channel.put(event);
- }
- // execute sink to process the events
- txn.commit();
- txn.close();
-
-
- checkRecordCountInTable(0, dbName, tblName);
- for (int i = 0; i < batchCount ; i++) {
- sink.process();
- }
- checkRecordCountInTable(totalRecords, dbName, tblName);
- sink.stop();
- checkRecordCountInTable(totalRecords, dbName, tblName);
- }
-
- @Test
- public void testSingleWriterSimplePartitionedTable() throws Exception {
- testSingleWriter(true, dbName, tblName, null);
- }
-
- @Test
- public void testSingleWriterSimpleUnPartitionedTable()
- throws Exception {
- TestUtil.dropDB(conf, dbName2);
- String dbLocation = dbFolder.newFolder(dbName2).getCanonicalPath() + ".db";
- dbLocation = dbLocation.replaceAll("\\\\","/"); // for windows paths
- TestUtil.createDbAndTable(driver, dbName2, tblName2, null, colNames2, colTypes2,
- null, dbLocation);
-
- try {
- testSingleWriter(false, dbName2, tblName2, null);
- } finally {
- TestUtil.dropDB(conf, dbName2);
- }
- }
-
- @Test
- public void testSingleWriterUseHeaders()
- throws Exception {
- String[] colNames = {COL1, COL2};
- String PART1_NAME = "country";
- String PART2_NAME = "hour";
- String[] partNames = {PART1_NAME, PART2_NAME};
- List<String> partitionVals = null;
- String PART1_VALUE = "%{" + PART1_NAME + "}";
- String PART2_VALUE = "%y-%m-%d-%k";
- partitionVals = new ArrayList<String>(2);
- partitionVals.add(PART1_VALUE);
- partitionVals.add(PART2_VALUE);
-
- String tblName = "hourlydata";
- TestUtil.dropDB(conf, dbName2);
- String dbLocation = dbFolder.newFolder(dbName2).getCanonicalPath() + ".db";
- dbLocation = dbLocation.replaceAll("\\\\","/"); // for windows paths
- TestUtil.createDbAndTable(driver, dbName2, tblName, partitionVals, colNames,
- colTypes, partNames, dbLocation);
-
- int totalRecords = 4;
- int batchSize = 2;
- int batchCount = totalRecords / batchSize;
-
- Context context = new Context();
- context.put("hive.metastore",metaStoreURI);
- context.put("hive.database",dbName2);
- context.put("hive.table",tblName);
- context.put("hive.partition", PART1_VALUE + "," + PART2_VALUE);
- context.put("autoCreatePartitions","true");
- context.put("useLocalTimeStamp", "false");
- context.put("batchSize","" + batchSize);
- context.put("serializer", HiveDelimitedTextSerializer.ALIAS);
- context.put("serializer.fieldnames", COL1 + ",," + COL2 + ",");
- context.put("heartBeatInterval", "0");
-
- Channel channel = startSink(sink, context);
-
- Calendar eventDate = Calendar.getInstance();
- List<String> bodies = Lists.newArrayList();
-
- // push events in two batches - two per batch. each batch is diff hour
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (int j = 1; j <= totalRecords; j++) {
- Event event = new SimpleEvent();
- String body = j + ",blah,This is a log message,other stuff";
- event.setBody(body.getBytes());
- eventDate.clear();
- eventDate.set(2014, 03, 03, j % batchCount, 1); // yy mm dd hh mm
- event.getHeaders().put( "timestamp",
- String.valueOf(eventDate.getTimeInMillis()) );
- event.getHeaders().put( PART1_NAME, "Asia" );
- bodies.add(body);
- channel.put(event);
- }
- // execute sink to process the events
- txn.commit();
- txn.close();
-
- checkRecordCountInTable(0, dbName2, tblName);
- for (int i = 0; i < batchCount ; i++) {
- sink.process();
- }
- checkRecordCountInTable(totalRecords, dbName2, tblName);
- sink.stop();
-
- // verify counters
- SinkCounter counter = sink.getCounter();
- Assert.assertEquals(2, counter.getConnectionCreatedCount());
- Assert.assertEquals(2, counter.getConnectionClosedCount());
- Assert.assertEquals(2, counter.getBatchCompleteCount());
- Assert.assertEquals(0, counter.getBatchEmptyCount());
- Assert.assertEquals(0, counter.getConnectionFailedCount() );
- Assert.assertEquals(4, counter.getEventDrainAttemptCount());
- Assert.assertEquals(4, counter.getEventDrainSuccessCount() );
-
- }
-
- @Test
- public void testHeartBeat()
- throws EventDeliveryException, IOException {
- int batchSize = 2;
- int batchCount = 3;
- int totalRecords = batchCount * batchSize;
- Context context = new Context();
- context.put("hive.metastore", metaStoreURI);
- context.put("hive.database", dbName);
- context.put("hive.table", tblName);
- context.put("hive.partition", PART1_VALUE + "," + PART2_VALUE);
- context.put("autoCreatePartitions","true");
- context.put("batchSize","" + batchSize);
- context.put("serializer", HiveDelimitedTextSerializer.ALIAS);
- context.put("serializer.fieldnames", COL1 + ",," + COL2 + ",");
- context.put("hive.txnsPerBatchAsk", "20");
- context.put("heartBeatInterval", "3"); // heartbeat in seconds
-
- Channel channel = startSink(sink, context);
-
- List<String> bodies = Lists.newArrayList();
-
- // push the events in two batches
- for (int i = 0; i < batchCount; i++) {
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (int j = 1; j <= batchSize; j++) {
- Event event = new SimpleEvent();
- String body = i * j + ",blah,This is a log message,other stuff";
- event.setBody(body.getBytes());
- bodies.add(body);
- channel.put(event);
- }
- // execute sink to process the events
- txn.commit();
- txn.close();
-
- sink.process();
- sleep(3000); // allow heartbeat to happen
- }
-
- sink.stop();
- checkRecordCountInTable(totalRecords, dbName, tblName);
- }
-
- @Test
- public void testJsonSerializer() throws Exception {
- int batchSize = 2;
- int batchCount = 2;
- int totalRecords = batchCount * batchSize;
- Context context = new Context();
- context.put("hive.metastore",metaStoreURI);
- context.put("hive.database",dbName);
- context.put("hive.table",tblName);
- context.put("hive.partition", PART1_VALUE + "," + PART2_VALUE);
- context.put("autoCreatePartitions","true");
- context.put("batchSize","" + batchSize);
- context.put("serializer", HiveJsonSerializer.ALIAS);
- context.put("serializer.fieldnames", COL1 + ",," + COL2 + ",");
- context.put("heartBeatInterval", "0");
-
- Channel channel = startSink(sink, context);
-
- List<String> bodies = Lists.newArrayList();
-
- // push the events in two batches
- for (int i = 0; i < batchCount; i++) {
- Transaction txn = channel.getTransaction();
- txn.begin();
- for (int j = 1; j <= batchSize; j++) {
- Event event = new SimpleEvent();
- String body = "{\"id\" : 1, \"msg\" : \"using json serializer\"}";
- event.setBody(body.getBytes());
- bodies.add(body);
- channel.put(event);
- }
- // execute sink to process the events
- txn.commit();
- txn.close();
-
- sink.process();
- }
- checkRecordCountInTable(totalRecords, dbName, tblName);
- sink.stop();
- checkRecordCountInTable(totalRecords, dbName, tblName);
- }
-
- @Test
- public void testErrorCounter() throws Exception {
- Channel channel = Mockito.mock(Channel.class);
- Mockito.when(channel.take()).thenThrow(new ChannelException("dummy"));
- Transaction transaction = Mockito.mock(BasicTransactionSemantics.class);
- Mockito.when(channel.getTransaction()).thenReturn(transaction);
-
- try {
- testSingleWriter(true, dbName, tblName, channel);
- } catch (EventDeliveryException e) {
- //Expected exception
- }
-
- SinkCounter sinkCounter = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
- Assert.assertEquals(1, sinkCounter.getChannelReadFail());
- }
-
- private void sleep(int n) {
- try {
- Thread.sleep(n);
- } catch (InterruptedException e) {
- }
- }
-
- private static Channel startSink(HiveSink sink, Context context) {
- return startSink(sink, context, null);
- }
-
- private static Channel startSink(HiveSink sink, Context context, Channel pChannel) {
- Configurables.configure(sink, context);
-
- Channel channel = pChannel == null ? new MemoryChannel() : pChannel;
- Configurables.configure(channel, context);
- sink.setChannel(channel);
- sink.start();
- return channel;
- }
-
- private void checkRecordCountInTable(int expectedCount, String db, String tbl)
- throws IOException {
- int count = TestUtil.listRecordsInTable(driver, db, tbl).size();
- Assert.assertEquals(expectedCount, count);
- }
-}
diff --git a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveWriter.java b/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveWriter.java
deleted file mode 100644
index 1e8ce1fec..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveWriter.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.sink.hive;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import junit.framework.Assert;
-import org.apache.flume.Context;
-import org.apache.flume.event.SimpleEvent;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class TestHiveWriter {
- static final String dbName = "testing";
- static final String tblName = "alerts";
-
- public static final String PART1_NAME = "continent";
- public static final String PART2_NAME = "country";
- public static final String[] partNames = { PART1_NAME, PART2_NAME };
-
- private static final String COL1 = "id";
- private static final String COL2 = "msg";
- final String[] colNames = {COL1,COL2};
- private String[] colTypes = { "int", "string" };
-
- private static final String PART1_VALUE = "Asia";
- private static final String PART2_VALUE = "India";
- private final ArrayList<String> partVals;
-
- private final String metaStoreURI;
-
- private HiveDelimitedTextSerializer serializer;
-
- private final HiveConf conf;
-
- private ExecutorService callTimeoutPool;
- int timeout = 10000; // msec
-
- @Rule
- public TemporaryFolder dbFolder = new TemporaryFolder();
-
- private final Driver driver;
-
- public TestHiveWriter() throws Exception {
- partVals = new ArrayList<String>(2);
- partVals.add(PART1_VALUE);
- partVals.add(PART2_VALUE);
-
- metaStoreURI = null;
-
- int callTimeoutPoolSize = 1;
- callTimeoutPool = Executors.newFixedThreadPool(callTimeoutPoolSize,
- new ThreadFactoryBuilder().setNameFormat("hiveWriterTest").build());
-
- // 1) Start metastore
- conf = new HiveConf(this.getClass());
- TestUtil.setConfValues(conf);
- if (metaStoreURI != null) {
- conf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreURI);
- }
-
- // 2) Setup Hive client
- SessionState.start(new CliSessionState(conf));
- driver = new Driver(conf);
-
- }
-
- @Before
- public void setUp() throws Exception {
- // 1) prepare hive
- TxnDbUtil.cleanDb(conf);
- TxnDbUtil.prepDb(conf);
-
- // 1) Setup tables
- TestUtil.dropDB(conf, dbName);
- String dbLocation = dbFolder.newFolder(dbName).getCanonicalPath() + ".db";
- dbLocation = dbLocation.replaceAll("\\\\","/"); // for windows paths
- TestUtil.createDbAndTable(driver, dbName, tblName, partVals, colNames, colTypes, partNames,
- dbLocation);
-
- // 2) Setup serializer
- Context ctx = new Context();
- ctx.put("serializer.fieldnames", COL1 + ",," + COL2 + ",");
- serializer = new HiveDelimitedTextSerializer();
- serializer.configure(ctx);
- }
-
- @Test
- public void testInstantiate() throws Exception {
- HiveEndPoint endPoint = new HiveEndPoint(metaStoreURI, dbName, tblName, partVals);
- SinkCounter sinkCounter = new SinkCounter(this.getClass().getName());
- HiveWriter writer = new HiveWriter(endPoint, 10, true, timeout, callTimeoutPool, "flumetest",
- serializer, sinkCounter);
-
- writer.close();
- }
-
- @Test
- public void testWriteBasic() throws Exception {
- HiveEndPoint endPoint = new HiveEndPoint(metaStoreURI, dbName, tblName, partVals);
- SinkCounter sinkCounter = new SinkCounter(this.getClass().getName());
- HiveWriter writer = new HiveWriter(endPoint, 10, true, timeout, callTimeoutPool, "flumetest",
- serializer, sinkCounter);
-
- writeEvents(writer,3);
- writer.flush(false);
- writer.close();
... 6117 lines suppressed ...