You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@logging.apache.org by rg...@apache.org on 2024/05/13 00:44:01 UTC

(logging-flume) branch trunk updated: Remove the hadoop components

This is an automated email from the ASF dual-hosted git repository.

rgoers pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/logging-flume.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 3e0984816 Remove the hadoop components
3e0984816 is described below

commit 3e098481668f0b47f0423be168448d3f7f5d8211
Author: Ralph Goers <rg...@apache.org>
AuthorDate: Sun May 12 16:48:28 2024 -0700

    Remove the hadoop components
---
 flume-bom/pom.xml                                  |   18 +-
 .../pom.xml                                        |   59 -
 .../HadoopCredentialStoreConfigFilter.java         |   77 -
 .../TestHadoopCredentialStoreConfigFilter.java     |  183 ---
 .../src/test/resources/test-password.txt           |    1 -
 .../src/test/resources/test-password2.txt          |    1 -
 flume-ng-configfilters/pom.xml                     |    1 -
 flume-ng-dist/pom.xml                              |   10 +-
 flume-ng-node/pom.xml                              |    5 +-
 flume-ng-sinks/flume-hdfs-sink/pom.xml             |  133 --
 .../apache/flume/sink/hdfs/AbstractHDFSWriter.java |  281 ----
 .../flume/sink/hdfs/AvroEventSerializer.java       |  211 ---
 .../flume/sink/hdfs/BucketClosedException.java     |   30 -
 .../org/apache/flume/sink/hdfs/BucketWriter.java   |  776 ---------
 .../flume/sink/hdfs/HDFSCompressedDataStream.java  |  162 --
 .../org/apache/flume/sink/hdfs/HDFSDataStream.java |  140 --
 .../org/apache/flume/sink/hdfs/HDFSEventSink.java  |  577 -------
 .../apache/flume/sink/hdfs/HDFSSequenceFile.java   |  122 --
 .../apache/flume/sink/hdfs/HDFSTextSerializer.java |   79 -
 .../flume/sink/hdfs/HDFSWritableSerializer.java    |   77 -
 .../org/apache/flume/sink/hdfs/HDFSWriter.java     |   47 -
 .../apache/flume/sink/hdfs/HDFSWriterFactory.java  |   43 -
 .../org/apache/flume/sink/hdfs/KerberosUser.java   |   72 -
 .../flume/sink/hdfs/SequenceFileSerializer.java    |   68 -
 .../sink/hdfs/SequenceFileSerializerFactory.java   |   90 --
 .../sink/hdfs/SequenceFileSerializerType.java      |   37 -
 .../apache/flume/sink/hdfs/HDFSBadDataStream.java  |   45 -
 .../apache/flume/sink/hdfs/HDFSTestSeqWriter.java  |   77 -
 .../flume/sink/hdfs/HDFSTestWriterFactory.java     |   42 -
 .../org/apache/flume/sink/hdfs/MockDataStream.java |   39 -
 .../org/apache/flume/sink/hdfs/MockFileSystem.java |  139 --
 .../flume/sink/hdfs/MockFsDataOutputStream.java    |   49 -
 .../org/apache/flume/sink/hdfs/MockHDFSWriter.java |  111 --
 .../apache/flume/sink/hdfs/MyCustomSerializer.java |   58 -
 .../flume/sink/hdfs/TestAvroEventSerializer.java   |  183 ---
 .../apache/flume/sink/hdfs/TestBucketWriter.java   |  719 ---------
 .../sink/hdfs/TestHDFSCompressedDataStream.java    |  141 --
 .../apache/flume/sink/hdfs/TestHDFSEventSink.java  | 1687 --------------------
 .../flume/sink/hdfs/TestHDFSEventSinkDeadlock.java |  127 --
 .../sink/hdfs/TestHDFSEventSinkOnMiniCluster.java  |  607 -------
 .../hdfs/TestSequenceFileSerializerFactory.java    |   58 -
 .../flume/sink/hdfs/TestUseRawLocalFileSystem.java |   95 --
 .../flume-hdfs-sink/src/test/resources/log4j2.xml  |   33 -
 flume-ng-sinks/flume-hive-sink/pom.xml             |  149 --
 .../java/org/apache/flume/sink/hive/Config.java    |   41 -
 .../sink/hive/HiveDelimitedTextSerializer.java     |  115 --
 .../flume/sink/hive/HiveEventSerializer.java       |   41 -
 .../apache/flume/sink/hive/HiveJsonSerializer.java |   62 -
 .../java/org/apache/flume/sink/hive/HiveSink.java  |  528 ------
 .../org/apache/flume/sink/hive/HiveWriter.java     |  513 ------
 .../org/apache/flume/sink/hive/TestHiveSink.java   |  413 -----
 .../org/apache/flume/sink/hive/TestHiveWriter.java |  350 ----
 .../java/org/apache/flume/sink/hive/TestUtil.java  |  235 ---
 .../flume-hive-sink/src/test/resources/log4j2.xml  |   33 -
 flume-ng-sinks/flume-ng-hbase2-sink/pom.xml        |  199 ---
 .../org/apache/flume/sink/hbase2/BatchAware.java   |   28 -
 .../flume/sink/hbase2/HBase2EventSerializer.java   |   61 -
 .../org/apache/flume/sink/hbase2/HBase2Sink.java   |  555 -------
 .../hbase2/HBase2SinkConfigurationConstants.java   |   77 -
 .../sink/hbase2/RegexHBase2EventSerializer.java    |  214 ---
 .../sink/hbase2/SimpleHBase2EventSerializer.java   |  151 --
 .../flume/sink/hbase2/SimpleRowKeyGenerator.java   |   46 -
 .../sink/hbase2/IncrementHBase2Serializer.java     |   83 -
 .../hbase2/MockSimpleHBase2EventSerializer.java    |   38 -
 .../apache/flume/sink/hbase2/TestHBase2Sink.java   |  786 ---------
 .../flume/sink/hbase2/TestHBase2SinkCreation.java  |   48 -
 .../hbase2/TestRegexHBase2EventSerializer.java     |  238 ---
 flume-ng-sinks/flume-ng-kudu-sink/pom.xml          |  195 ---
 .../sink/kudu/AvroKuduOperationsProducer.java      |  282 ----
 .../flume/sink/kudu/KuduOperationsProducer.java    |   54 -
 .../java/org/apache/flume/sink/kudu/KuduSink.java  |  315 ----
 .../sink/kudu/KuduSinkConfigurationConstants.java  |   77 -
 .../sink/kudu/RegexpKuduOperationsProducer.java    |  398 -----
 .../kudu/SimpleKeyedKuduOperationsProducer.java    |  136 --
 .../sink/kudu/SimpleKuduOperationsProducer.java    |   92 --
 .../apache/flume/sink/kudu/KuduSinkTestUtil.java   |   99 --
 .../sink/kudu/TestAvroKuduOperationsProducer.java  |  183 ---
 .../sink/kudu/TestKeyedKuduOperationsProducer.java |  189 ---
 .../org/apache/flume/sink/kudu/TestKuduSink.java   |  213 ---
 .../kudu/TestRegexpKuduOperationsProducer.java     |  184 ---
 ...TestRegexpKuduOperationsProducerParseError.java |  291 ----
 .../apache/flume/sink/kudu/TestSecureKuduSink.java |  126 --
 .../src/test/resources/log4j2.xml                  |   32 -
 .../resources/testAvroKuduOperationsProducer.avsc  |   11 -
 flume-ng-sinks/pom.xml                             |   24 -
 85 files changed, 17 insertions(+), 15398 deletions(-)

diff --git a/flume-bom/pom.xml b/flume-bom/pom.xml
index ee4e7bf23..2d803c846 100644
--- a/flume-bom/pom.xml
+++ b/flume-bom/pom.xml
@@ -32,7 +32,7 @@ limitations under the License.
   <name>Apache Flume Artifacts</name>
 
   <properties>
-    <flume-hadoop.version>1.11.1-SNAPSHOT</flume-hadoop.version>
+    <flume-hadoop.version>2.0.0-SNAPSHOT</flume-hadoop.version>
     <flume-http.version>1.11.1-SNAPSHOT</flume-http.version>
     <flume-irc.version>1.11.1-SNAPSHOT</flume-irc.version>
     <flume-jdbc.version>2.0.0-SNAPSHOT</flume-jdbc.version>
@@ -92,8 +92,8 @@ limitations under the License.
       </dependency>
       <dependency>
         <groupId>org.apache.flume</groupId>
-        <artifactId>flume-ng-hadoop-credential-store-config-filter</artifactId>
-        <version>${project.version}</version>
+        <artifactId>flume-hadoop-credential-store-config-filter</artifactId>
+        <version>${flume-hadoop.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.flume</groupId>
@@ -172,12 +172,12 @@ limitations under the License.
         <version>${morphline.version}</version>
       </dependency>
       <dependency>
-        <groupId>org.apache.flume.flume-ng-sinks</groupId>
+        <groupId>org.apache.flume</groupId>
         <artifactId>flume-hdfs-sink</artifactId>
         <version>${flume-hadoop.version}</version>
       </dependency>
       <dependency>
-        <groupId>org.apache.flume.flume-ng-sinks</groupId>
+        <groupId>org.apache.flume</groupId>
         <artifactId>flume-hive-sink</artifactId>
         <version>${flume-hadoop.version}</version>
       </dependency>
@@ -192,8 +192,8 @@ limitations under the License.
         <version>${flume-irc.version}</version>
       </dependency>
       <dependency>
-        <groupId>org.apache.flume.flume-ng-sinks</groupId>
-        <artifactId>flume-ng-hbase2-sink</artifactId>
+        <groupId>org.apache.flume</groupId>
+        <artifactId>flume-hbase2-sink</artifactId>
         <version>${flume-hadoop.version}</version>
       </dependency>
       <dependency>
@@ -202,8 +202,8 @@ limitations under the License.
         <version>${flume-kafka.version}</version>
       </dependency>
       <dependency>
-        <groupId>org.apache.flume.flume-ng-sinks</groupId>
-        <artifactId>flume-ng-kudu-sink</artifactId>
+        <groupId>org.apache.flume</groupId>
+        <artifactId>flume-kudu-sink</artifactId>
         <version>${flume-hadoop.version}</version>
       </dependency>
       <dependency>
diff --git a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/pom.xml b/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/pom.xml
deleted file mode 100644
index a744bb20f..000000000
--- a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/pom.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements.  See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>flume-ng-configfilters</artifactId>
-        <groupId>org.apache.flume</groupId>
-        <version>1.11.1-SNAPSHOT</version>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-
-    <properties>
-        <module.name>org.apache.flume.configfilter.credstore</module.name>
-    </properties>
-
-    <artifactId>flume-ng-hadoop-credential-store-config-filter</artifactId>
-    <dependencies>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-common</artifactId>
-            <optional>true</optional>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.flume</groupId>
-            <artifactId>flume-ng-config-filter-api</artifactId>
-        </dependency>
-        <!-- test dependencies -->
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.github.stefanbirkner</groupId>
-            <artifactId>system-rules</artifactId>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-
-</project>
\ No newline at end of file
diff --git a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/main/java/org/apache/flume/configfilter/HadoopCredentialStoreConfigFilter.java b/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/main/java/org/apache/flume/configfilter/HadoopCredentialStoreConfigFilter.java
deleted file mode 100644
index c8d68703c..000000000
--- a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/main/java/org/apache/flume/configfilter/HadoopCredentialStoreConfigFilter.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.flume.configfilter;
-
-import java.io.IOException;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HadoopCredentialStoreConfigFilter extends AbstractConfigFilter {
-
-  private static final Logger LOGGER = LoggerFactory.getLogger(
-      HadoopCredentialStoreConfigFilter.class);
-
-  static final String PASSWORD_FILE_CONFIG_KEY
-      = "credstore.java-keystore-provider.password-file";
-  static final String CREDENTIAL_PROVIDER_PATH
-      = "credential.provider.path";
-  static final String HADOOP_SECURITY = "hadoop.security.";
-  private Configuration hadoopConfiguration;
-
-  public void initializeWithConfiguration(Map<String, String> configuration) {
-
-    LOGGER.debug("Initializing hadoop credential store.");
-    hadoopConfiguration = new Configuration();
-    hadoopConfiguration.set(
-        HADOOP_SECURITY + CREDENTIAL_PROVIDER_PATH,
-        configuration.get(CREDENTIAL_PROVIDER_PATH)
-    );
-
-    String passwordFile = configuration.get(PASSWORD_FILE_CONFIG_KEY);
-    if (passwordFile != null && !passwordFile.isEmpty()) {
-      checkPasswordFile(passwordFile);
-      hadoopConfiguration.set(
-          HADOOP_SECURITY + PASSWORD_FILE_CONFIG_KEY, passwordFile
-      );
-    }
-  }
-
-  private void checkPasswordFile(String passwordFile) {
-    if (Thread.currentThread().getContextClassLoader().getResource(passwordFile) == null) {
-      LOGGER.error("The java keystore provider password file has to be on the classpath." +
-          " The password file provided in the configuration cannot be found and will not be used"
-      );
-    }
-  }
-
-  @Override
-  public String filter(String key) {
-    char[] result = null;
-    try {
-      result = hadoopConfiguration.getPassword(key);
-    } catch (IOException e) {
-      LOGGER.error("Error while reading value for key {}: ", key, e);
-    }
-
-    return result == null ? null : new String(result);
-  }
-
-
-}
diff --git a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/java/org/apache/flume/configfilter/TestHadoopCredentialStoreConfigFilter.java b/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/java/org/apache/flume/configfilter/TestHadoopCredentialStoreConfigFilter.java
deleted file mode 100644
index 55f74d8cf..000000000
--- a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/java/org/apache/flume/configfilter/TestHadoopCredentialStoreConfigFilter.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.flume.configfilter;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.alias.CredentialShell;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.contrib.java.lang.system.EnvironmentVariables;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.util.HashMap;
-
-import static org.apache.flume.configfilter.HadoopCredentialStoreConfigFilter.CREDENTIAL_PROVIDER_PATH;
-import static org.apache.flume.configfilter.HadoopCredentialStoreConfigFilter.HADOOP_SECURITY;
-import static org.apache.flume.configfilter.HadoopCredentialStoreConfigFilter.PASSWORD_FILE_CONFIG_KEY;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-
-public class TestHadoopCredentialStoreConfigFilter {
-
-  private static String providerPathDefault;
-  private static String providerPathEnv;
-  private static String providerPathPwdFile;
-
-  @ClassRule
-  public static final EnvironmentVariables environmentVariables
-      = new EnvironmentVariables();
-  private static File fileDefault;
-  private static File fileEnvPassword;
-  private static File fileFilePassword;
-  private HadoopCredentialStoreConfigFilter configFilter;
-
-  @BeforeClass
-  public static void setUpClass() throws Exception {
-    generateTempFileNames();
-    fillCredStoreWithDefaultPassword();
-    fillCredStoreWithPasswordFile();
-    fillCredStoreWithEnvironmentVariablePassword();
-  }
-  @AfterClass
-  public static void tearDown() {
-    fileDefault.deleteOnExit();
-    fileEnvPassword.deleteOnExit();
-    fileFilePassword.deleteOnExit();
-  }
-
-  @Before
-  public void setUp() {
-    String[] objects = System.getenv().keySet().toArray(new String[0]);
-    environmentVariables.clear(objects);
-    configFilter = new HadoopCredentialStoreConfigFilter();
-  }
-
-  @Test
-  public void filterDefaultPasswordFile() {
-    HashMap<String, String> configuration = new HashMap<>();
-    configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathDefault);
-    configFilter.initializeWithConfiguration(configuration);
-
-    assertEquals("filtered_default", configFilter.filter("password"));
-  }
-
-  @Test
-  public void filterWithEnvPassword() {
-    environmentVariables.set("HADOOP_CREDSTORE_PASSWORD","envSecret");
-    HashMap<String, String> configuration = new HashMap<>();
-    configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathEnv);
-    configFilter.initializeWithConfiguration(configuration);
-
-    assertEquals("filtered_env", configFilter.filter("password"));
-  }
-
-  @Test
-  public void filterWithPasswordFile() {
-    HashMap<String, String> configuration = new HashMap<>();
-    configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathPwdFile);
-    configuration.put(PASSWORD_FILE_CONFIG_KEY, "test-password.txt");
-    configFilter.initializeWithConfiguration(configuration);
-
-    assertEquals("filtered_file", configFilter.filter("password"));
-  }
-
-  @Test
-  public void filterWithEnvNoPassword() {
-    HashMap<String, String> configuration = new HashMap<>();
-    configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathEnv);
-    configFilter.initializeWithConfiguration(configuration);
-
-    assertNull(configFilter.filter("password"));
-  }
-
-  @Test
-  public void filterErrorWithPasswordFileWrongPassword() {
-    HashMap<String, String> configuration = new HashMap<>();
-    configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathPwdFile);
-    configuration.put(PASSWORD_FILE_CONFIG_KEY, "test-password2.txt");
-    configFilter.initializeWithConfiguration(configuration);
-
-    assertNull(configFilter.filter("password"));
-  }
-
-  @Test
-  public void filterErrorWithPasswordFileNoPasswordFile() {
-    HashMap<String, String> configuration = new HashMap<>();
-    configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathPwdFile);
-    configFilter.initializeWithConfiguration(configuration);
-
-    assertNull(configFilter.filter("password"));
-  }
-
-  @Test(expected = IllegalArgumentException.class)
-  public void filterErrorWithNoProvider() {
-    HashMap<String, String> configuration = new HashMap<>();
-    configFilter.initializeWithConfiguration(configuration);
-  }
-
-
-  private static void fillCredStoreWithEnvironmentVariablePassword() throws Exception {
-    environmentVariables.set("HADOOP_CREDSTORE_PASSWORD", "envSecret");
-
-    runCommand("create password -value filtered_env -provider "
-        + providerPathEnv, new Configuration());
-  }
-
-  private static void fillCredStoreWithPasswordFile() throws Exception {
-    Configuration conf = new Configuration();
-    conf.set(
-        HADOOP_SECURITY + PASSWORD_FILE_CONFIG_KEY,
-        "test-password.txt"
-    );
-    runCommand("create password -value filtered_file -provider "
-        + providerPathPwdFile, conf);
-  }
-
-  private static void fillCredStoreWithDefaultPassword() throws Exception {
-    runCommand("create password -value filtered_default -provider "
-        + providerPathDefault, new Configuration());
-  }
-
-  private static void generateTempFileNames() throws IOException {
-    fileDefault = Files.createTempFile("test-default-pwd-", ".jceks").toFile();
-    boolean deleted = fileDefault.delete();
-    fileEnvPassword = Files.createTempFile("test-env-pwd-", ".jceks").toFile();
-    deleted &= fileEnvPassword.delete();
-    fileFilePassword = Files.createTempFile("test-file-pwd-", ".jceks").toFile();
-    deleted &= fileFilePassword.delete();
-    if (!deleted) {
-      fail("Could not delete temporary files");
-    }
-
-    providerPathDefault = "jceks://file/" + fileDefault.getAbsolutePath();
-    providerPathEnv = "jceks://file/" + fileEnvPassword.getAbsolutePath();
-    providerPathPwdFile = "jceks://file/" + fileFilePassword.getAbsolutePath();
-  }
-
-  private static void runCommand(String c, Configuration conf) throws Exception {
-    ToolRunner.run(conf, new CredentialShell(), c.split(" "));
-  }
-
-
-}
\ No newline at end of file
diff --git a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/resources/test-password.txt b/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/resources/test-password.txt
deleted file mode 100644
index db5fd9185..000000000
--- a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/resources/test-password.txt
+++ /dev/null
@@ -1 +0,0 @@
-fileSecret
\ No newline at end of file
diff --git a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/resources/test-password2.txt b/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/resources/test-password2.txt
deleted file mode 100644
index 2dee0b131..000000000
--- a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/src/test/resources/test-password2.txt
+++ /dev/null
@@ -1 +0,0 @@
-fileSecret2
\ No newline at end of file
diff --git a/flume-ng-configfilters/pom.xml b/flume-ng-configfilters/pom.xml
index 1ed177500..7cc35884a 100644
--- a/flume-ng-configfilters/pom.xml
+++ b/flume-ng-configfilters/pom.xml
@@ -31,7 +31,6 @@ limitations under the License.
 
     <modules>
         <module>flume-ng-environment-variable-config-filter</module>
-        <module>flume-ng-hadoop-credential-store-config-filter</module>
         <module>flume-ng-config-filter-api</module>
         <module>flume-ng-external-process-config-filter</module>
     </modules>
diff --git a/flume-ng-dist/pom.xml b/flume-ng-dist/pom.xml
index 7be10b2f1..373b499fe 100644
--- a/flume-ng-dist/pom.xml
+++ b/flume-ng-dist/pom.xml
@@ -133,7 +133,7 @@
       <artifactId>flume-kafka-channel</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.apache.flume.flume-ng-sinks</groupId>
+      <groupId>org.apache.flume</groupId>
       <artifactId>flume-hdfs-sink</artifactId>
     </dependency>
     <dependency>
@@ -141,8 +141,8 @@
       <artifactId>flume-irc-sink</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.apache.flume.flume-ng-sinks</groupId>
-      <artifactId>flume-ng-hbase2-sink</artifactId>
+      <groupId>org.apache.flume</groupId>
+      <artifactId>flume-hbase2-sink</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.flume.flume-ng-sinks</groupId>
@@ -157,7 +157,7 @@
       <artifactId>flume-kafka-sink</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.apache.flume.flume-ng-sinks</groupId>
+      <groupId>org.apache.flume</groupId>
       <artifactId>flume-hive-sink</artifactId>
     </dependency>
     <dependency>
@@ -194,7 +194,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.flume</groupId>
-      <artifactId>flume-ng-hadoop-credential-store-config-filter</artifactId>
+      <artifactId>flume-hadoop-credential-store-config-filter</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.flume</groupId>
diff --git a/flume-ng-node/pom.xml b/flume-ng-node/pom.xml
index 00d77e2f6..2f7453f66 100644
--- a/flume-ng-node/pom.xml
+++ b/flume-ng-node/pom.xml
@@ -56,10 +56,11 @@
       <artifactId>flume-ng-core</artifactId>
     </dependency>
 
+    <!--
     <dependency>
-      <groupId>org.apache.flume.flume-ng-sinks</groupId>
+      <groupId>org.apache.flume</groupId>
       <artifactId>flume-hdfs-sink</artifactId>
-    </dependency>
+    </dependency>-->
 
     <dependency>
       <groupId>org.apache.flume.flume-ng-sinks</groupId>
diff --git a/flume-ng-sinks/flume-hdfs-sink/pom.xml b/flume-ng-sinks/flume-hdfs-sink/pom.xml
deleted file mode 100644
index 07404c1cc..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/pom.xml
+++ /dev/null
@@ -1,133 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements.  See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-  <modelVersion>4.0.0</modelVersion>
-
-  <parent>
-    <artifactId>flume-ng-sinks</artifactId>
-    <groupId>org.apache.flume</groupId>
-    <version>1.11.1-SNAPSHOT</version>
-  </parent>
-
-  <groupId>org.apache.flume.flume-ng-sinks</groupId>
-  <artifactId>flume-hdfs-sink</artifactId>
-  <name>Flume NG HDFS Sink</name>
-
-  <properties>
-    <!-- TODO fix spotbugs/pmd violations -->
-    <spotbugs.maxAllowedViolations>14</spotbugs.maxAllowedViolations>
-    <pmd.maxAllowedViolations>25</pmd.maxAllowedViolations>
-    <module.name>org.apache.flume.sink.hdfs</module.name>
-  </properties>
-
-  <dependencies>
-
-    <dependency>
-      <groupId>org.apache.flume</groupId>
-      <artifactId>flume-ng-sdk</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.flume</groupId>
-      <artifactId>flume-ng-configuration</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.flume</groupId>
-      <artifactId>flume-ng-core</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-api</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-slf4j-impl</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-1.2-api</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-all</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <optional>true</optional>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-      <optional>true</optional>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs-client</artifactId>
-      <optional>true</optional>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-auth</artifactId>
-      <optional>true</optional>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-minicluster</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>commons-lang</groupId>
-      <artifactId>commons-lang</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>commons-io</groupId>
-      <artifactId>commons-io</artifactId>
-    </dependency>
-
-  </dependencies>
-
-</project>
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AbstractHDFSWriter.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AbstractHDFSWriter.java
deleted file mode 100644
index d2a07faf1..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AbstractHDFSWriter.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import com.google.common.base.Preconditions;
-import org.apache.flume.Context;
-import org.apache.flume.FlumeException;
-import org.apache.flume.annotations.InterfaceAudience;
-import org.apache.flume.annotations.InterfaceStability;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public abstract class AbstractHDFSWriter implements HDFSWriter {
-
-  private static final Logger logger =
-      LoggerFactory.getLogger(AbstractHDFSWriter.class);
-
-  private FSDataOutputStream outputStream;
-  private FileSystem fs;
-  private Path destPath;
-  private Method refGetNumCurrentReplicas = null;
-  private Method refGetDefaultReplication = null;
-  private Method refHflushOrSync = null;
-  private Integer configuredMinReplicas = null;
-  private Integer numberOfCloseRetries = null;
-  private long timeBetweenCloseRetries = Long.MAX_VALUE;
-
-  static final Object[] NO_ARGS = new Object[]{};
-
-  @Override
-  public void configure(Context context) {
-    configuredMinReplicas = context.getInteger("hdfs.minBlockReplicas");
-    if (configuredMinReplicas != null) {
-      Preconditions.checkArgument(configuredMinReplicas >= 0,
-          "hdfs.minBlockReplicas must be greater than or equal to 0");
-    }
-    numberOfCloseRetries = context.getInteger("hdfs.closeTries", 1) - 1;
-
-    if (numberOfCloseRetries > 1) {
-      try {
-        //hdfs.callTimeout is deprecated from 1.9
-        timeBetweenCloseRetries = context.getLong("hdfs.callTimeout", 30000L);
-      } catch (NumberFormatException e) {
-        logger.warn("hdfs.callTimeout can not be parsed to a long: " +
-                    context.getLong("hdfs.callTimeout"));
-      }
-      timeBetweenCloseRetries = Math.max(timeBetweenCloseRetries / numberOfCloseRetries, 1000);
-    }
-
-  }
-
-  /**
-   * Contract for subclasses: Call registerCurrentStream() on open,
-   * unregisterCurrentStream() on close, and the base class takes care of the
-   * rest.
-   * @return
-   */
-  @Override
-  public boolean isUnderReplicated() {
-    try {
-      int numBlocks = getNumCurrentReplicas();
-      if (numBlocks == -1) {
-        return false;
-      }
-      int desiredBlocks;
-      if (configuredMinReplicas != null) {
-        desiredBlocks = configuredMinReplicas;
-      } else {
-        desiredBlocks = getFsDesiredReplication();
-      }
-      return numBlocks < desiredBlocks;
-    } catch (IllegalAccessException e) {
-      logger.error("Unexpected error while checking replication factor", e);
-    } catch (InvocationTargetException e) {
-      logger.error("Unexpected error while checking replication factor", e);
-    } catch (IllegalArgumentException e) {
-      logger.error("Unexpected error while checking replication factor", e);
-    }
-    return false;
-  }
-
-  protected void registerCurrentStream(FSDataOutputStream outputStream,
-                                      FileSystem fs, Path destPath) {
-    Preconditions.checkNotNull(outputStream, "outputStream must not be null");
-    Preconditions.checkNotNull(fs, "fs must not be null");
-    Preconditions.checkNotNull(destPath, "destPath must not be null");
-
-    this.outputStream = outputStream;
-    this.fs = fs;
-    this.destPath = destPath;
-    this.refGetNumCurrentReplicas = reflectGetNumCurrentReplicas(outputStream);
-    this.refGetDefaultReplication = reflectGetDefaultReplication(fs);
-    this.refHflushOrSync = reflectHflushOrSync(outputStream);
-
-  }
-
-  protected void unregisterCurrentStream() {
-    this.outputStream = null;
-    this.fs = null;
-    this.destPath = null;
-    this.refGetNumCurrentReplicas = null;
-    this.refGetDefaultReplication = null;
-  }
-
-  public int getFsDesiredReplication() {
-    short replication = 0;
-    if (fs != null && destPath != null) {
-      if (refGetDefaultReplication != null) {
-        try {
-          replication = (Short) refGetDefaultReplication.invoke(fs, destPath);
-        } catch (IllegalAccessException e) {
-          logger.warn("Unexpected error calling getDefaultReplication(Path)", e);
-        } catch (InvocationTargetException e) {
-          logger.warn("Unexpected error calling getDefaultReplication(Path)", e);
-        }
-      } else {
-        // will not work on Federated HDFS (see HADOOP-8014)
-        replication = fs.getDefaultReplication();
-      }
-    }
-    return replication;
-  }
-
-  /**
-   * This method gets the datanode replication count for the current open file.
-   *
-   * If the pipeline isn't started yet or is empty, you will get the default
-   * replication factor.
-   *
-   * <p/>If this function returns -1, it means you
-   * are not properly running with the HDFS-826 patch.
-   * @throws InvocationTargetException
-   * @throws IllegalAccessException
-   * @throws IllegalArgumentException
-   */
-  public int getNumCurrentReplicas()
-      throws IllegalArgumentException, IllegalAccessException,
-          InvocationTargetException {
-    if (refGetNumCurrentReplicas != null && outputStream != null) {
-      OutputStream dfsOutputStream = outputStream.getWrappedStream();
-      if (dfsOutputStream != null) {
-        Object repl = refGetNumCurrentReplicas.invoke(dfsOutputStream, NO_ARGS);
-        if (repl instanceof Integer) {
-          return ((Integer)repl).intValue();
-        }
-      }
-    }
-    return -1;
-  }
-
-  /**
-   * Find the 'getNumCurrentReplicas' on the passed <code>os</code> stream.
-   * @return Method or null.
-   */
-  private Method reflectGetNumCurrentReplicas(FSDataOutputStream os) {
-    Method m = null;
-    if (os != null) {
-      Class<? extends OutputStream> wrappedStreamClass = os.getWrappedStream()
-          .getClass();
-      try {
-        m = wrappedStreamClass.getDeclaredMethod("getNumCurrentReplicas",
-            new Class<?>[] {});
-        m.setAccessible(true);
-      } catch (NoSuchMethodException e) {
-        logger.info("FileSystem's output stream doesn't support"
-            + " getNumCurrentReplicas; --HDFS-826 not available; fsOut="
-            + wrappedStreamClass.getName() + "; err=" + e);
-      } catch (SecurityException e) {
-        logger.info("Doesn't have access to getNumCurrentReplicas on "
-            + "FileSystems's output stream --HDFS-826 not available; fsOut="
-            + wrappedStreamClass.getName(), e);
-        m = null; // could happen on setAccessible()
-      }
-    }
-    if (m != null) {
-      logger.debug("Using getNumCurrentReplicas--HDFS-826");
-    }
-    return m;
-  }
-
-  /**
-   * Find the 'getDefaultReplication' method on the passed <code>fs</code>
-   * FileSystem that takes a Path argument.
-   * @return Method or null.
-   */
-  private Method reflectGetDefaultReplication(FileSystem fileSystem) {
-    Method m = null;
-    if (fileSystem != null) {
-      Class<?> fsClass = fileSystem.getClass();
-      try {
-        m = fsClass.getMethod("getDefaultReplication",
-            new Class<?>[] { Path.class });
-      } catch (NoSuchMethodException e) {
-        logger.debug("FileSystem implementation doesn't support"
-            + " getDefaultReplication(Path); -- HADOOP-8014 not available; " +
-            "className = " + fsClass.getName() + "; err = " + e);
-      } catch (SecurityException e) {
-        logger.debug("No access to getDefaultReplication(Path) on "
-            + "FileSystem implementation -- HADOOP-8014 not available; " +
-            "className = " + fsClass.getName() + "; err = " + e);
-      }
-    }
-    if (m != null) {
-      logger.debug("Using FileSystem.getDefaultReplication(Path) from " +
-          "HADOOP-8014");
-    }
-    return m;
-  }
-
-  private Method reflectHflushOrSync(FSDataOutputStream os) {
-    Method m = null;
-    if (os != null) {
-      Class<?> fsDataOutputStreamClass = os.getClass();
-      try {
-        m = fsDataOutputStreamClass.getMethod("hflush");
-      } catch (NoSuchMethodException ex) {
-        logger.debug("HFlush not found. Will use sync() instead");
-        try {
-          m = fsDataOutputStreamClass.getMethod("sync");
-        } catch (Exception ex1) {
-          String msg = "Neither hflush not sync were found. That seems to be " +
-              "a problem!";
-          logger.error(msg);
-          throw new FlumeException(msg, ex1);
-        }
-      }
-    }
-    return m;
-  }
-
-  /**
-   * If hflush is available in this version of HDFS, then this method calls
-   * hflush, else it calls sync.
-   * @param os - The stream to flush/sync
-   * @throws IOException
-   */
-  protected void hflushOrSync(FSDataOutputStream os) throws IOException {
-    try {
-      // At this point the refHflushOrSync cannot be null,
-      // since register method would have thrown if it was.
-      this.refHflushOrSync.invoke(os);
-    } catch (InvocationTargetException e) {
-      String msg = "Error while trying to hflushOrSync!";
-      logger.error(msg);
-      Throwable cause = e.getCause();
-      if (cause != null && cause instanceof IOException) {
-        throw (IOException)cause;
-      }
-      throw new FlumeException(msg, e);
-    } catch (Exception e) {
-      String msg = "Error while trying to hflushOrSync!";
-      logger.error(msg);
-      throw new FlumeException(msg, e);
-    }
-  }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AvroEventSerializer.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AvroEventSerializer.java
deleted file mode 100644
index 323174240..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AvroEventSerializer.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import org.apache.avro.AvroRuntimeException;
-import org.apache.avro.Schema;
-import org.apache.avro.file.CodecFactory;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.avro.io.DatumWriter;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.FlumeException;
-import org.apache.flume.conf.Configurable;
-import org.apache.flume.serialization.EventSerializer;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URL;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-
-import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.COMPRESSION_CODEC;
-import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_COMPRESSION_CODEC;
-import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_STATIC_SCHEMA_URL;
-import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_SYNC_INTERVAL_BYTES;
-import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.STATIC_SCHEMA_URL;
-import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.SYNC_INTERVAL_BYTES;
-
-/**
- * <p>
- * This class serializes Flume {@linkplain org.apache.flume.Event events} into Avro data files. The
- * Flume event body is read as an Avro datum, and is then written to the
- * {@link org.apache.flume.serialization.EventSerializer}'s output stream in Avro data file format.
- * </p>
- * <p>
- * The Avro schema is determined by reading a Flume event header. The schema may be
- * specified either as a literal, by setting {@link #AVRO_SCHEMA_LITERAL_HEADER} (not
- * recommended, since the full schema must be transmitted in every event),
- * or as a URL which the schema may be read from, by setting {@link
- * #AVRO_SCHEMA_URL_HEADER}. Schemas read from URLs are cached by instances of this
- * class so that the overhead of retrieval is minimized.
- * </p>
- */
-public class AvroEventSerializer implements EventSerializer, Configurable {
-
-  private static final Logger logger =
-      LoggerFactory.getLogger(AvroEventSerializer.class);
-
-  public static final String AVRO_SCHEMA_LITERAL_HEADER = "flume.avro.schema.literal";
-  public static final String AVRO_SCHEMA_URL_HEADER = "flume.avro.schema.url";
-
-  private final OutputStream out;
-  private DatumWriter<Object> writer = null;
-  private DataFileWriter<Object> dataFileWriter = null;
-
-  private int syncIntervalBytes;
-  private String compressionCodec;
-  private Map<String, Schema> schemaCache = new HashMap<String, Schema>();
-  private String staticSchemaURL;
-
-  private AvroEventSerializer(OutputStream out) {
-    this.out = out;
-  }
-
-  @Override
-  public void configure(Context context) {
-    syncIntervalBytes =
-        context.getInteger(SYNC_INTERVAL_BYTES, DEFAULT_SYNC_INTERVAL_BYTES);
-    compressionCodec =
-        context.getString(COMPRESSION_CODEC, DEFAULT_COMPRESSION_CODEC);
-    staticSchemaURL = context.getString(STATIC_SCHEMA_URL, DEFAULT_STATIC_SCHEMA_URL);
-  }
-
-  @Override
-  public void afterCreate() throws IOException {
-    // no-op
-  }
-
-  @Override
-  public void afterReopen() throws IOException {
-    // impossible to initialize DataFileWriter without writing the schema?
-    throw new UnsupportedOperationException("Avro API doesn't support append");
-  }
-
-  @Override
-  public void write(Event event) throws IOException {
-    if (dataFileWriter == null) {
-      initialize(event);
-    }
-    dataFileWriter.appendEncoded(ByteBuffer.wrap(event.getBody()));
-  }
-
-  private void initialize(Event event) throws IOException {
-    Schema schema = null;
-    String schemaUrl = event.getHeaders().get(AVRO_SCHEMA_URL_HEADER);
-    String schemaString = event.getHeaders().get(AVRO_SCHEMA_LITERAL_HEADER);
-
-    if (schemaUrl != null) { // if URL_HEADER is there then use it
-      schema = schemaCache.get(schemaUrl);
-      if (schema == null) {
-        schema = loadFromUrl(schemaUrl);
-        schemaCache.put(schemaUrl, schema);
-      }
-    } else if (schemaString != null) { // fallback to LITERAL_HEADER if it was there
-      schema = new Schema.Parser().parse(schemaString);
-    } else if (staticSchemaURL != null) {   // fallback to static url if it was there
-      schema = schemaCache.get(staticSchemaURL);
-      if (schema == null) {
-        schema = loadFromUrl(staticSchemaURL);
-        schemaCache.put(staticSchemaURL, schema);
-      }
-    } else { // no other options so giving up
-      throw new FlumeException("Could not find schema for event " + event);
-    }
-
-    writer = new GenericDatumWriter<Object>(schema);
-    dataFileWriter = new DataFileWriter<Object>(writer);
-
-    dataFileWriter.setSyncInterval(syncIntervalBytes);
-
-    try {
-      CodecFactory codecFactory = CodecFactory.fromString(compressionCodec);
-      dataFileWriter.setCodec(codecFactory);
-    } catch (AvroRuntimeException e) {
-      logger.warn("Unable to instantiate avro codec with name (" +
-          compressionCodec + "). Compression disabled. Exception follows.", e);
-    }
-
-    dataFileWriter.create(schema, out);
-  }
-
-  private Schema loadFromUrl(String schemaUrl) throws IOException {
-    Configuration conf = new Configuration();
-    Schema.Parser parser = new Schema.Parser();
-    if (schemaUrl.toLowerCase(Locale.ENGLISH).startsWith("hdfs://")) {
-      FileSystem fs = FileSystem.get(conf);
-      FSDataInputStream input = null;
-      try {
-        input = fs.open(new Path(schemaUrl));
-        return parser.parse(input);
-      } finally {
-        if (input != null) {
-          input.close();
-        }
-      }
-    } else {
-      InputStream is = null;
-      try {
-        is = new URL(schemaUrl).openStream();
-        return parser.parse(is);
-      } finally {
-        if (is != null) {
-          is.close();
-        }
-      }
-    }
-  }
-
-  @Override
-  public void flush() throws IOException {
-    dataFileWriter.flush();
-  }
-
-  @Override
-  public void beforeClose() throws IOException {
-    // no-op
-  }
-
-  @Override
-  public boolean supportsReopen() {
-    return false;
-  }
-
-  public static class Builder implements EventSerializer.Builder {
-
-    @Override
-    public EventSerializer build(Context context, OutputStream out) {
-      AvroEventSerializer writer = new AvroEventSerializer(out);
-      writer.configure(context);
-      return writer;
-    }
-
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketClosedException.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketClosedException.java
deleted file mode 100644
index 1d8a9e4c5..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketClosedException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import org.apache.flume.FlumeException;
-
-public class BucketClosedException extends FlumeException {
-
-  private static final long serialVersionUID = -4216667125119540357L;
-
-  public BucketClosedException(String msg) {
-    super(msg);
-  }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketWriter.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketWriter.java
deleted file mode 100644
index c3941a692..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketWriter.java
+++ /dev/null
@@ -1,776 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Throwables;
-import org.apache.flume.Clock;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.SystemClock;
-import org.apache.flume.auth.PrivilegedExecutor;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.flume.sink.hdfs.HDFSEventSink.WriterCallback;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.security.PrivilegedExceptionAction;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Internal API intended for HDFSSink use.
- * This class does file rolling and handles file formats and serialization.
- * Only the public methods in this class are thread safe.
- */
-class BucketWriter {
-
-  private static final Logger LOG = LoggerFactory
-      .getLogger(BucketWriter.class);
-
-  /**
-   * This lock ensures that only one thread can open a file at a time.
-   */
-  private static final Integer staticLock = new Integer(1);
-  private Method isClosedMethod = null;
-
-  private final HDFSWriter writer;
-  private final long rollInterval;
-  private final long rollSize;
-  private final long rollCount;
-  private final long batchSize;
-  private final CompressionCodec codeC;
-  private final CompressionType compType;
-  private final ScheduledExecutorService timedRollerPool;
-  private final PrivilegedExecutor proxyUser;
-
-  private final AtomicLong fileExtensionCounter;
-
-  private long eventCounter;
-  private long processSize;
-
-  private FileSystem fileSystem;
-
-  private volatile String filePath;
-  private volatile String fileName;
-  private volatile String inUsePrefix;
-  private volatile String inUseSuffix;
-  private volatile String fileSuffix;
-  private volatile String bucketPath;
-  private volatile String targetPath;
-  private volatile long batchCounter;
-  private volatile boolean isOpen;
-  private volatile boolean isUnderReplicated;
-  private volatile int consecutiveUnderReplRotateCount = 0;
-  private volatile ScheduledFuture<Void> timedRollFuture;
-  private SinkCounter sinkCounter;
-  private final int idleTimeout;
-  private volatile ScheduledFuture<Void> idleFuture;
-  private final WriterCallback onCloseCallback;
-  private final String onCloseCallbackPath;
-  private final long callTimeout;
-  private final ExecutorService callTimeoutPool;
-  private final int maxConsecUnderReplRotations = 30; // make this config'able?
-
-  private boolean mockFsInjected = false;
-
-  private final long retryInterval;
-  private final int maxRetries;
-
-  // flag that the bucket writer was closed due to idling and thus shouldn't be
-  // reopened. Not ideal, but avoids internals of owners
-  protected AtomicBoolean closed = new AtomicBoolean();
-  AtomicInteger renameTries = new AtomicInteger(0);
-
-  BucketWriter(long rollInterval, long rollSize, long rollCount, long batchSize,
-      Context context, String filePath, String fileName, String inUsePrefix,
-      String inUseSuffix, String fileSuffix, CompressionCodec codeC,
-      CompressionType compType, HDFSWriter writer,
-      ScheduledExecutorService timedRollerPool, PrivilegedExecutor proxyUser,
-      SinkCounter sinkCounter, int idleTimeout, WriterCallback onCloseCallback,
-      String onCloseCallbackPath, long callTimeout,
-      ExecutorService callTimeoutPool, long retryInterval,
-      int maxRetries) {
-    this(rollInterval, rollSize, rollCount, batchSize,
-            context, filePath, fileName, inUsePrefix,
-            inUseSuffix, fileSuffix, codeC,
-            compType, writer,
-            timedRollerPool, proxyUser,
-            sinkCounter, idleTimeout, onCloseCallback,
-            onCloseCallbackPath, callTimeout,
-            callTimeoutPool, retryInterval,
-            maxRetries, new SystemClock());
-  }
-
-  BucketWriter(long rollInterval, long rollSize, long rollCount, long batchSize,
-           Context context, String filePath, String fileName, String inUsePrefix,
-           String inUseSuffix, String fileSuffix, CompressionCodec codeC,
-           CompressionType compType, HDFSWriter writer,
-           ScheduledExecutorService timedRollerPool, PrivilegedExecutor proxyUser,
-           SinkCounter sinkCounter, int idleTimeout, WriterCallback onCloseCallback,
-           String onCloseCallbackPath, long callTimeout,
-           ExecutorService callTimeoutPool, long retryInterval,
-           int maxRetries, Clock clock) {
-    this.rollInterval = rollInterval;
-    this.rollSize = rollSize;
-    this.rollCount = rollCount;
-    this.batchSize = batchSize;
-    this.filePath = filePath;
-    this.fileName = fileName;
-    this.inUsePrefix = inUsePrefix;
-    this.inUseSuffix = inUseSuffix;
-    this.fileSuffix = fileSuffix;
-    this.codeC = codeC;
-    this.compType = compType;
-    this.writer = writer;
-    this.timedRollerPool = timedRollerPool;
-    this.proxyUser = proxyUser;
-    this.sinkCounter = sinkCounter;
-    this.idleTimeout = idleTimeout;
-    this.onCloseCallback = onCloseCallback;
-    this.onCloseCallbackPath = onCloseCallbackPath;
-    this.callTimeout = callTimeout;
-    this.callTimeoutPool = callTimeoutPool;
-    fileExtensionCounter = new AtomicLong(clock.currentTimeMillis());
-
-    this.retryInterval = retryInterval;
-    this.maxRetries = maxRetries;
-    isOpen = false;
-    isUnderReplicated = false;
-    this.writer.configure(context);
-  }
-
-  @VisibleForTesting
-  void setFileSystem(FileSystem fs) {
-    this.fileSystem = fs;
-    mockFsInjected = true;
-  }
-
-  /**
-   * Clear the class counters
-   */
-  private void resetCounters() {
-    eventCounter = 0;
-    processSize = 0;
-    batchCounter = 0;
-  }
-
-  private Method getRefIsClosed() {
-    try {
-      return fileSystem.getClass().getMethod("isFileClosed",
-        Path.class);
-    } catch (Exception e) {
-      LOG.info("isFileClosed() is not available in the version of the " +
-               "distributed filesystem being used. " +
-               "Flume will not attempt to re-close files if the close fails " +
-               "on the first attempt");
-      return null;
-    }
-  }
-
-  private Boolean isFileClosed(FileSystem fs, Path tmpFilePath) throws Exception {
-    return (Boolean)(isClosedMethod.invoke(fs, tmpFilePath));
-  }
-
-  /**
-   * open() is called by append()
-   * @throws IOException
-   * @throws InterruptedException
-   */
-  private void open() throws IOException, InterruptedException {
-    if ((filePath == null) || (writer == null)) {
-      throw new IOException("Invalid file settings");
-    }
-
-    final Configuration config = new Configuration();
-    // disable FileSystem JVM shutdown hook
-    config.setBoolean("fs.automatic.close", false);
-
-    // Hadoop is not thread safe when doing certain RPC operations,
-    // including getFileSystem(), when running under Kerberos.
-    // open() must be called by one thread at a time in the JVM.
-    // NOTE: tried synchronizing on the underlying Kerberos principal previously
-    // which caused deadlocks. See FLUME-1231.
-    synchronized (staticLock) {
-      checkAndThrowInterruptedException();
-
-      try {
-        long counter = fileExtensionCounter.incrementAndGet();
-
-        String fullFileName = fileName + "." + counter;
-
-        if (fileSuffix != null && fileSuffix.length() > 0) {
-          fullFileName += fileSuffix;
-        } else if (codeC != null) {
-          fullFileName += codeC.getDefaultExtension();
-        }
-
-        bucketPath = filePath + "/" + inUsePrefix
-          + fullFileName + inUseSuffix;
-        targetPath = filePath + "/" + fullFileName;
-
-        LOG.info("Creating " + bucketPath);
-        callWithTimeout(new CallRunner<Void>() {
-          @Override
-          public Void call() throws Exception {
-            if (codeC == null) {
-              // Need to get reference to FS using above config before underlying
-              // writer does in order to avoid shutdown hook &
-              // IllegalStateExceptions
-              if (!mockFsInjected) {
-                fileSystem = new Path(bucketPath).getFileSystem(config);
-              }
-              writer.open(bucketPath);
-            } else {
-              // need to get reference to FS before writer does to
-              // avoid shutdown hook
-              if (!mockFsInjected) {
-                fileSystem = new Path(bucketPath).getFileSystem(config);
-              }
-              writer.open(bucketPath, codeC, compType);
-            }
-            return null;
-          }
-        });
-      } catch (Exception ex) {
-        sinkCounter.incrementConnectionFailedCount();
-        if (ex instanceof IOException) {
-          throw (IOException) ex;
-        } else {
-          throw Throwables.propagate(ex);
-        }
-      }
-    }
-    isClosedMethod = getRefIsClosed();
-    sinkCounter.incrementConnectionCreatedCount();
-    resetCounters();
-
-    // if time-based rolling is enabled, schedule the roll
-    if (rollInterval > 0) {
-      Callable<Void> action = new Callable<Void>() {
-        public Void call() throws Exception {
-          LOG.debug("Rolling file ({}): Roll scheduled after {} sec elapsed.",
-              bucketPath, rollInterval);
-          try {
-            // Roll the file and remove reference from sfWriters map.
-            close(true);
-          } catch (Throwable t) {
-            LOG.error("Unexpected error", t);
-          }
-          return null;
-        }
-      };
-      timedRollFuture = timedRollerPool.schedule(action, rollInterval,
-          TimeUnit.SECONDS);
-    }
-
-    isOpen = true;
-  }
-
-  /**
-   * Close the file handle and rename the temp file to the permanent filename.
-   * Safe to call multiple times. Logs HDFSWriter.close() exceptions. This
-   * method will not cause the bucket writer to be dereferenced from the HDFS
-   * sink that owns it. This method should be used only when size or count
-   * based rolling closes this file.
-   */
-  public void close() throws InterruptedException {
-    close(false);
-  }
-
-  private CallRunner<Void> createCloseCallRunner() {
-    return new CallRunner<Void>() {
-      @Override
-      public Void call() throws Exception {
-        writer.close(); // could block
-        return null;
-      }
-    };
-  }
-
-  private class CloseHandler implements Callable<Void> {
-    private final String path = bucketPath;
-    private int closeTries = 0;
-
-    @Override
-    public Void call() throws Exception {
-      close(false);
-      return null;
-    }
-
-    /**
-     * Tries to close the writer. Repeats the close if the maximum number
-     * of retries is not reached or an immediate close is not reuqested.
-     * If all close attempts were unsuccessful we try to recover the lease.
-     * @param immediate An immediate close is required
-     */
-    public void close(boolean immediate) {
-      closeTries++;
-      boolean shouldRetry = closeTries < maxRetries && !immediate;
-      try {
-        callWithTimeout(createCloseCallRunner());
-        sinkCounter.incrementConnectionClosedCount();
-      } catch (InterruptedException | IOException e) {
-        LOG.warn("Closing file: " + path + " failed. Will " +
-            "retry again in " + retryInterval + " seconds.", e);
-        if (timedRollerPool != null && !timedRollerPool.isTerminated()) {
-          if (shouldRetry) {
-            timedRollerPool.schedule(this, retryInterval, TimeUnit.SECONDS);
-          }
-        } else {
-          LOG.warn("Cannot retry close any more timedRollerPool is null or terminated");
-        }
-        if (!shouldRetry) {
-          LOG.warn("Unsuccessfully attempted to close " + path + " " +
-                  maxRetries + " times. Initializing lease recovery.");
-          sinkCounter.incrementConnectionFailedCount();
-          recoverLease();
-        }
-      }
-    }
-  }
-
-  private class ScheduledRenameCallable implements Callable<Void> {
-    private final String path = bucketPath;
-    private final String finalPath = targetPath;
-    private FileSystem fs = fileSystem;
-    private int renameTries = 1; // one attempt is already done
-
-    @Override
-    public Void call() throws Exception {
-      if (renameTries >= maxRetries) {
-        LOG.warn("Unsuccessfully attempted to rename " + path + " " +
-                maxRetries + " times. File may still be open.");
-        return null;
-      }
-      renameTries++;
-      try {
-        renameBucket(path, finalPath, fs);
-      } catch (Exception e) {
-        LOG.warn("Renaming file: " + path + " failed. Will " +
-            "retry again in " + retryInterval + " seconds.", e);
-        timedRollerPool.schedule(this, retryInterval, TimeUnit.SECONDS);
-        return null;
-      }
-      return null;
-    }
-  }
-
-  /**
-   * Tries to start the lease recovery process for the current bucketPath
-   * if the fileSystem is DistributedFileSystem.
-   * Catches and logs the IOException.
-   */
-  private synchronized void recoverLease() {
-    if (bucketPath != null && fileSystem instanceof DistributedFileSystem) {
-      try {
-        LOG.debug("Starting lease recovery for {}", bucketPath);
-        ((DistributedFileSystem) fileSystem).recoverLease(new Path(bucketPath));
-      } catch (IOException ex) {
-        LOG.warn("Lease recovery failed for {}", bucketPath, ex);
-      }
-    }
-  }
-
-  public void close(boolean callCloseCallback) throws InterruptedException {
-    close(callCloseCallback, false);
-  }
-
-  /**
-   * Close the file handle and rename the temp file to the permanent filename.
-   * Safe to call multiple times. Logs HDFSWriter.close() exceptions.
-   */
-  public void close(boolean callCloseCallback, boolean immediate)
-      throws InterruptedException {
-    if (callCloseCallback) {
-      if (closed.compareAndSet(false, true)) {
-        runCloseAction(); //remove from the cache as soon as possible
-      } else {
-        LOG.warn("This bucketWriter is already closing or closed.");
-      }
-    }
-    doClose(immediate);
-  }
-
-  private synchronized void doClose(boolean immediate)
-      throws InterruptedException {
-    checkAndThrowInterruptedException();
-    try {
-      flush();
-    } catch (IOException e) {
-      LOG.warn("pre-close flush failed", e);
-    }
-
-    LOG.info("Closing {}", bucketPath);
-    if (isOpen) {
-      new CloseHandler().close(immediate);
-      isOpen = false;
-    } else {
-      LOG.info("HDFSWriter is already closed: {}", bucketPath);
-    }
-
-    // NOTE: timed rolls go through this codepath as well as other roll types
-    if (timedRollFuture != null && !timedRollFuture.isDone()) {
-      timedRollFuture.cancel(false); // do not cancel myself if running!
-      timedRollFuture = null;
-    }
-
-    if (idleFuture != null && !idleFuture.isDone()) {
-      idleFuture.cancel(false); // do not cancel myself if running!
-      idleFuture = null;
-    }
-
-    if (bucketPath != null && fileSystem != null) {
-      // could block or throw IOException
-      try {
-        renameBucket(bucketPath, targetPath, fileSystem);
-      } catch (Exception e) {
-        LOG.warn("failed to rename() file (" + bucketPath +
-                 "). Exception follows.", e);
-        sinkCounter.incrementConnectionFailedCount();
-        final Callable<Void> scheduledRename =  new ScheduledRenameCallable();
-        timedRollerPool.schedule(scheduledRename, retryInterval, TimeUnit.SECONDS);
-      }
-    }
-  }
-
-  /**
-   * flush the data
-   * @throws IOException
-   * @throws InterruptedException
-   */
-  public synchronized void flush() throws IOException, InterruptedException {
-    checkAndThrowInterruptedException();
-    if (!isBatchComplete()) {
-      doFlush();
-
-      if (idleTimeout > 0) {
-        // if the future exists and couldn't be cancelled, that would mean it has already run
-        // or been cancelled
-        if (idleFuture == null || idleFuture.cancel(false)) {
-          Callable<Void> idleAction = new Callable<Void>() {
-            public Void call() throws Exception {
-              LOG.info("Closing idle bucketWriter {} at {}", bucketPath,
-                       System.currentTimeMillis());
-              if (isOpen) {
-                close(true);
-              }
-              return null;
-            }
-          };
-          idleFuture = timedRollerPool.schedule(idleAction, idleTimeout,
-              TimeUnit.SECONDS);
-        }
-      }
-    }
-  }
-
-  private void runCloseAction() {
-    try {
-      if (onCloseCallback != null) {
-        onCloseCallback.run(onCloseCallbackPath);
-      }
-    } catch (Throwable t) {
-      LOG.error("Unexpected error", t);
-    }
-  }
-
-  /**
-   * doFlush() must only be called by flush()
-   * @throws IOException
-   */
-  private void doFlush() throws IOException, InterruptedException {
-    callWithTimeout(new CallRunner<Void>() {
-      @Override
-      public Void call() throws Exception {
-        writer.sync(); // could block
-        return null;
-      }
-    });
-    batchCounter = 0;
-  }
-
-  /**
-   * Open file handles, write data, update stats, handle file rolling and
-   * batching / flushing. <br />
-   * If the write fails, the file is implicitly closed and then the IOException
-   * is rethrown. <br />
-   * We rotate before append, and not after, so that the active file rolling
-   * mechanism will never roll an empty file. This also ensures that the file
-   * creation time reflects when the first event was written.
-   *
-   * @throws IOException
-   * @throws InterruptedException
-   */
-  public synchronized void append(final Event event)
-          throws IOException, InterruptedException {
-    checkAndThrowInterruptedException();
-    // If idleFuture is not null, cancel it before we move forward to avoid a
-    // close call in the middle of the append.
-    if (idleFuture != null) {
-      idleFuture.cancel(false);
-      // There is still a small race condition - if the idleFuture is already
-      // running, interrupting it can cause HDFS close operation to throw -
-      // so we cannot interrupt it while running. If the future could not be
-      // cancelled, it is already running - wait for it to finish before
-      // attempting to write.
-      if (!idleFuture.isDone()) {
-        try {
-          idleFuture.get(callTimeout, TimeUnit.MILLISECONDS);
-        } catch (TimeoutException ex) {
-          LOG.warn("Timeout while trying to cancel closing of idle file. Idle" +
-                   " file close may have failed", ex);
-        } catch (Exception ex) {
-          LOG.warn("Error while trying to cancel closing of idle file. ", ex);
-        }
-      }
-      idleFuture = null;
-    }
-
-    // If the bucket writer was closed due to roll timeout or idle timeout,
-    // force a new bucket writer to be created. Roll count and roll size will
-    // just reuse this one
-    if (!isOpen) {
-      if (closed.get()) {
-        throw new BucketClosedException("This bucket writer was closed and " +
-          "this handle is thus no longer valid");
-      }
-      open();
-    }
-
-    // check if it's time to rotate the file
-    if (shouldRotate()) {
-      boolean doRotate = true;
-
-      if (isUnderReplicated) {
-        if (maxConsecUnderReplRotations > 0 &&
-            consecutiveUnderReplRotateCount >= maxConsecUnderReplRotations) {
-          doRotate = false;
-          if (consecutiveUnderReplRotateCount == maxConsecUnderReplRotations) {
-            LOG.error("Hit max consecutive under-replication rotations ({}); " +
-                "will not continue rolling files under this path due to " +
-                "under-replication", maxConsecUnderReplRotations);
-          }
-        } else {
-          LOG.warn("Block Under-replication detected. Rotating file.");
-        }
-        consecutiveUnderReplRotateCount++;
-      } else {
-        consecutiveUnderReplRotateCount = 0;
-      }
-
-      if (doRotate) {
-        close();
-        open();
-      }
-    }
-
-    // write the event
-    try {
-      sinkCounter.incrementEventDrainAttemptCount();
-      callWithTimeout(new CallRunner<Void>() {
-        @Override
-        public Void call() throws Exception {
-          writer.append(event); // could block
-          return null;
-        }
-      });
-    } catch (IOException e) {
-      LOG.warn("Caught IOException writing to HDFSWriter ({}). Closing file (" +
-          bucketPath + ") and rethrowing exception.",
-          e.getMessage());
-      close(true);
-      throw e;
-    }
-
-    // update statistics
-    processSize += event.getBody().length;
-    eventCounter++;
-    batchCounter++;
-
-    if (batchCounter == batchSize) {
-      flush();
-    }
-  }
-
-  /**
-   * check if time to rotate the file
-   */
-  private boolean shouldRotate() {
-    boolean doRotate = false;
-
-    if (writer.isUnderReplicated()) {
-      this.isUnderReplicated = true;
-      doRotate = true;
-    } else {
-      this.isUnderReplicated = false;
-    }
-
-    if ((rollCount > 0) && (rollCount <= eventCounter)) {
-      LOG.debug("rolling: rollCount: {}, events: {}", rollCount, eventCounter);
-      doRotate = true;
-    }
-
-    if ((rollSize > 0) && (rollSize <= processSize)) {
-      LOG.debug("rolling: rollSize: {}, bytes: {}", rollSize, processSize);
-      doRotate = true;
-    }
-
-    return doRotate;
-  }
-
-  /**
-   * Rename bucketPath file from .tmp to permanent location.
-   */
-  // When this bucket writer is rolled based on rollCount or
-  // rollSize, the same instance is reused for the new file. But if
-  // the previous file was not closed/renamed,
-  // the bucket writer fields no longer point to it and hence need
-  // to be passed in from the thread attempting to close it. Even
-  // when the bucket writer is closed due to close timeout,
-  // this method can get called from the scheduled thread so the
-  // file gets closed later - so an implicit reference to this
-  // bucket writer would still be alive in the Callable instance.
-  private void renameBucket(String bucketPath, String targetPath, final FileSystem fs)
-      throws IOException, InterruptedException {
-    if (bucketPath.equals(targetPath)) {
-      return;
-    }
-
-    final Path srcPath = new Path(bucketPath);
-    final Path dstPath = new Path(targetPath);
-
-    callWithTimeout(new CallRunner<Void>() {
-      @Override
-      public Void call() throws Exception {
-        if (fs.exists(srcPath)) { // could block
-          LOG.info("Renaming " + srcPath + " to " + dstPath);
-          renameTries.incrementAndGet();
-          fs.rename(srcPath, dstPath); // could block
-        }
-        return null;
-      }
-    });
-  }
-
-  @Override
-  public String toString() {
-    return "[ " + this.getClass().getSimpleName() + " targetPath = " + targetPath +
-        ", bucketPath = " + bucketPath + " ]";
-  }
-
-  private boolean isBatchComplete() {
-    return (batchCounter == 0);
-  }
-
-  /**
-   * This method if the current thread has been interrupted and throws an
-   * exception.
-   * @throws InterruptedException
-   */
-  private static void checkAndThrowInterruptedException()
-          throws InterruptedException {
-    if (Thread.currentThread().interrupted()) {
-      throw new InterruptedException("Timed out before HDFS call was made. "
-              + "Your hdfs.callTimeout might be set too low or HDFS calls are "
-              + "taking too long.");
-    }
-  }
-
-  /**
-   * Execute the callable on a separate thread and wait for the completion
-   * for the specified amount of time in milliseconds. In case of timeout
-   * cancel the callable and throw an IOException
-   */
-  private <T> T callWithTimeout(final CallRunner<T> callRunner)
-      throws IOException, InterruptedException {
-    Future<T> future = callTimeoutPool.submit(new Callable<T>() {
-      @Override
-      public T call() throws Exception {
-        return proxyUser.execute(new PrivilegedExceptionAction<T>() {
-          @Override
-          public T run() throws Exception {
-            return callRunner.call();
-          }
-        });
-      }
-    });
-    try {
-      if (callTimeout > 0) {
-        return future.get(callTimeout, TimeUnit.MILLISECONDS);
-      } else {
-        return future.get();
-      }
-    } catch (TimeoutException eT) {
-      future.cancel(true);
-      sinkCounter.incrementConnectionFailedCount();
-      throw new IOException("Callable timed out after " +
-        callTimeout + " ms" + " on file: " + bucketPath, eT);
-    } catch (ExecutionException e1) {
-      sinkCounter.incrementConnectionFailedCount();
-      Throwable cause = e1.getCause();
-      if (cause instanceof IOException) {
-        throw (IOException) cause;
-      } else if (cause instanceof InterruptedException) {
-        throw (InterruptedException) cause;
-      } else if (cause instanceof RuntimeException) {
-        throw (RuntimeException) cause;
-      } else if (cause instanceof Error) {
-        throw (Error)cause;
-      } else {
-        throw new RuntimeException(e1);
-      }
-    } catch (CancellationException ce) {
-      throw new InterruptedException(
-        "Blocked callable interrupted by rotation event");
-    } catch (InterruptedException ex) {
-      LOG.warn("Unexpected Exception " + ex.getMessage(), ex);
-      throw ex;
-    }
-  }
-
-  /**
-   * Simple interface whose <tt>call</tt> method is called by
-   * {#callWithTimeout} in a new thread inside a
-   * {@linkplain java.security.PrivilegedExceptionAction#run()} call.
-   * @param <T>
-   */
-  private interface CallRunner<T> {
-    T call() throws Exception;
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSCompressedDataStream.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSCompressedDataStream.java
deleted file mode 100644
index 80b7cb4c3..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSCompressedDataStream.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.serialization.EventSerializer;
-import org.apache.flume.serialization.EventSerializerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CodecPool;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionOutputStream;
-import org.apache.hadoop.io.compress.Compressor;
-import org.apache.hadoop.io.compress.DefaultCodec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HDFSCompressedDataStream extends AbstractHDFSWriter {
-
-  private static final Logger logger =
-      LoggerFactory.getLogger(HDFSCompressedDataStream.class);
-
-  private FSDataOutputStream fsOut;
-  private CompressionOutputStream cmpOut;
-  private boolean isFinished = false;
-
-  private String serializerType;
-  private Context serializerContext;
-  private EventSerializer serializer;
-  private boolean useRawLocalFileSystem;
-  private Compressor compressor;
-
-  @Override
-  public void configure(Context context) {
-    super.configure(context);
-
-    serializerType = context.getString("serializer", "TEXT");
-    useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
-        false);
-    serializerContext = new Context(
-        context.getSubProperties(EventSerializer.CTX_PREFIX));
-    logger.info("Serializer = " + serializerType + ", UseRawLocalFileSystem = "
-        + useRawLocalFileSystem);
-  }
-
-  @Override
-  public void open(String filePath) throws IOException {
-    DefaultCodec defCodec = new DefaultCodec();
-    CompressionType cType = CompressionType.BLOCK;
-    open(filePath, defCodec, cType);
-  }
-
-  @Override
-  public void open(String filePath, CompressionCodec codec,
-      CompressionType cType) throws IOException {
-    Configuration conf = new Configuration();
-    Path dstPath = new Path(filePath);
-    FileSystem hdfs = dstPath.getFileSystem(conf);
-    if (useRawLocalFileSystem) {
-      if (hdfs instanceof LocalFileSystem) {
-        hdfs = ((LocalFileSystem)hdfs).getRaw();
-      } else {
-        logger.warn("useRawLocalFileSystem is set to true but file system " +
-            "is not of type LocalFileSystem: " + hdfs.getClass().getName());
-      }
-    }
-    boolean appending = false;
-    if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) {
-      fsOut = hdfs.append(dstPath);
-      appending = true;
-    } else {
-      fsOut = hdfs.create(dstPath);
-    }
-    if (compressor == null) {
-      compressor = CodecPool.getCompressor(codec, conf);
-    }
-    cmpOut = codec.createOutputStream(fsOut, compressor);
-    serializer = EventSerializerFactory.getInstance(serializerType,
-        serializerContext, cmpOut);
-    if (appending && !serializer.supportsReopen()) {
-      cmpOut.close();
-      serializer = null;
-      throw new IOException("serializer (" + serializerType
-          + ") does not support append");
-    }
-
-    registerCurrentStream(fsOut, hdfs, dstPath);
-
-    if (appending) {
-      serializer.afterReopen();
-    } else {
-      serializer.afterCreate();
-    }
-    isFinished = false;
-  }
-
-  @Override
-  public void append(Event e) throws IOException {
-    if (isFinished) {
-      cmpOut.resetState();
-      isFinished = false;
-    }
-    serializer.write(e);
-  }
-
-  @Override
-  public void sync() throws IOException {
-    // We must use finish() and resetState() here -- flush() is apparently not
-    // supported by the compressed output streams (it's a no-op).
-    // Also, since resetState() writes headers, avoid calling it without an
-    // additional write/append operation.
-    // Note: There are bugs in Hadoop & JDK w/ pure-java gzip; see HADOOP-8522.
-    serializer.flush();
-    if (!isFinished) {
-      cmpOut.finish();
-      isFinished = true;
-    }
-    fsOut.flush();
-    hflushOrSync(this.fsOut);
-  }
-
-  @Override
-  public void close() throws IOException {
-    serializer.flush();
-    serializer.beforeClose();
-    if (!isFinished) {
-      cmpOut.finish();
-      isFinished = true;
-    }
-    fsOut.flush();
-    hflushOrSync(fsOut);
-    cmpOut.close();
-    if (compressor != null) {
-      CodecPool.returnCompressor(compressor);
-      compressor = null;
-    }
-    unregisterCurrentStream();
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSDataStream.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSDataStream.java
deleted file mode 100644
index c4ad9192d..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSDataStream.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.serialization.EventSerializer;
-import org.apache.flume.serialization.EventSerializerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HDFSDataStream extends AbstractHDFSWriter {
-
-  private static final Logger logger = LoggerFactory.getLogger(HDFSDataStream.class);
-
-  private FSDataOutputStream outStream;
-  private String serializerType;
-  private Context serializerContext;
-  private EventSerializer serializer;
-  private boolean useRawLocalFileSystem;
-
-  @Override
-  public void configure(Context context) {
-    super.configure(context);
-
-    serializerType = context.getString("serializer", "TEXT");
-    useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
-        false);
-    serializerContext =
-        new Context(context.getSubProperties(EventSerializer.CTX_PREFIX));
-    logger.info("Serializer = " + serializerType + ", UseRawLocalFileSystem = "
-        + useRawLocalFileSystem);
-  }
-
-  @VisibleForTesting
-  protected FileSystem getDfs(Configuration conf, Path dstPath) throws IOException {
-    return dstPath.getFileSystem(conf);
-  }
-
-  protected void doOpen(Configuration conf, Path dstPath, FileSystem hdfs) throws IOException {
-    if (useRawLocalFileSystem) {
-      if (hdfs instanceof LocalFileSystem) {
-        hdfs = ((LocalFileSystem)hdfs).getRaw();
-      } else {
-        logger.warn("useRawLocalFileSystem is set to true but file system " +
-            "is not of type LocalFileSystem: " + hdfs.getClass().getName());
-      }
-    }
-
-    boolean appending = false;
-    if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) {
-      outStream = hdfs.append(dstPath);
-      appending = true;
-    } else {
-      outStream = hdfs.create(dstPath);
-    }
-
-    serializer = EventSerializerFactory.getInstance(
-        serializerType, serializerContext, outStream);
-    if (appending && !serializer.supportsReopen()) {
-      outStream.close();
-      serializer = null;
-      throw new IOException("serializer (" + serializerType +
-          ") does not support append");
-    }
-
-    // must call superclass to check for replication issues
-    registerCurrentStream(outStream, hdfs, dstPath);
-
-    if (appending) {
-      serializer.afterReopen();
-    } else {
-      serializer.afterCreate();
-    }
-  }
-
-  @Override
-  public void open(String filePath) throws IOException {
-    Configuration conf = new Configuration();
-    Path dstPath = new Path(filePath);
-    FileSystem hdfs = getDfs(conf, dstPath);
-    doOpen(conf, dstPath, hdfs);
-  }
-
-  @Override
-  public void open(String filePath, CompressionCodec codec,
-                   CompressionType cType) throws IOException {
-    open(filePath);
-  }
-
-  @Override
-  public void append(Event e) throws IOException {
-    serializer.write(e);
-  }
-
-  @Override
-  public void sync() throws IOException {
-    serializer.flush();
-    outStream.flush();
-    hflushOrSync(outStream);
-  }
-
-  @Override
-  public void close() throws IOException {
-    serializer.flush();
-    serializer.beforeClose();
-    outStream.flush();
-    hflushOrSync(outStream);
-    outStream.close();
-
-    unregisterCurrentStream();
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSEventSink.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSEventSink.java
deleted file mode 100644
index 9e64eb532..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSEventSink.java
+++ /dev/null
@@ -1,577 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.Map.Entry;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.flume.Channel;
-import org.apache.flume.Clock;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.SystemClock;
-import org.apache.flume.Transaction;
-import org.apache.flume.auth.FlumeAuthenticationUtil;
-import org.apache.flume.auth.PrivilegedExecutor;
-import org.apache.flume.conf.BatchSizeSupported;
-import org.apache.flume.conf.Configurable;
-import org.apache.flume.formatter.output.BucketPath;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.flume.sink.AbstractSink;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-public class HDFSEventSink extends AbstractSink implements Configurable, BatchSizeSupported {
-  public interface WriterCallback {
-    public void run(String filePath);
-  }
-
-  private static final Logger LOG = LoggerFactory.getLogger(HDFSEventSink.class);
-
-  private static String DIRECTORY_DELIMITER = System.getProperty("file.separator");
-
-  private static final long defaultRollInterval = 30;
-  private static final long defaultRollSize = 1024;
-  private static final long defaultRollCount = 10;
-  private static final String defaultFileName = "FlumeData";
-  private static final String defaultSuffix = "";
-  private static final String defaultInUsePrefix = "";
-  private static final String defaultInUseSuffix = ".tmp";
-  private static final long defaultBatchSize = 100;
-  private static final String defaultFileType = HDFSWriterFactory.SequenceFileType;
-  private static final int defaultMaxOpenFiles = 5000;
-  // Time between close retries, in seconds
-  private static final long defaultRetryInterval = 180;
-  // Retry forever.
-  private static final int defaultTryCount = Integer.MAX_VALUE;
-
-  public static final String IN_USE_SUFFIX_PARAM_NAME = "hdfs.inUseSuffix";
-
-  /**
-   * Default length of time we wait for blocking BucketWriter calls
-   * before timing out the operation. Intended to prevent server hangs.
-   */
-  private static final long defaultCallTimeout = 30000;
-  /**
-   * Default number of threads available for tasks
-   * such as append/open/close/flush with hdfs.
-   * These tasks are done in a separate thread in
-   * the case that they take too long. In which
-   * case we create a new file and move on.
-   */
-  private static final int defaultThreadPoolSize = 10;
-  private static final int defaultRollTimerPoolSize = 1;
-
-  private final HDFSWriterFactory writerFactory;
-  private WriterLinkedHashMap sfWriters;
-
-  private long rollInterval;
-  private long rollSize;
-  private long rollCount;
-  private long batchSize;
-  private int threadsPoolSize;
-  private int rollTimerPoolSize;
-  private CompressionCodec codeC;
-  private CompressionType compType;
-  private String fileType;
-  private String filePath;
-  private String fileName;
-  private String suffix;
-  private String inUsePrefix;
-  private String inUseSuffix;
-  private TimeZone timeZone;
-  private int maxOpenFiles;
-  private ExecutorService callTimeoutPool;
-  private ScheduledExecutorService timedRollerPool;
-
-  private boolean needRounding = false;
-  private int roundUnit = Calendar.SECOND;
-  private int roundValue = 1;
-  private boolean useLocalTime = false;
-
-  private long callTimeout;
-  private Context context;
-  private SinkCounter sinkCounter;
-
-  private volatile int idleTimeout;
-  private Clock clock;
-  private FileSystem mockFs;
-  private HDFSWriter mockWriter;
-  private final Object sfWritersLock = new Object();
-  private long retryInterval;
-  private int tryCount;
-  private PrivilegedExecutor privExecutor;
-
-  /*
-   * Extended Java LinkedHashMap for open file handle LRU queue.
-   * We want to clear the oldest file handle if there are too many open ones.
-   */
-  private static class WriterLinkedHashMap
-      extends LinkedHashMap<String, BucketWriter> {
-
-    private final int maxOpenFiles;
-
-    public WriterLinkedHashMap(int maxOpenFiles) {
-      super(16, 0.75f, true); // stock initial capacity/load, access ordering
-      this.maxOpenFiles = maxOpenFiles;
-    }
-
-    @Override
-    protected boolean removeEldestEntry(Entry<String, BucketWriter> eldest) {
-      if (size() > maxOpenFiles) {
-        // If we have more that max open files, then close the last one and
-        // return true
-        try {
-          eldest.getValue().close();
-        } catch (InterruptedException e) {
-          LOG.warn(eldest.getKey().toString(), e);
-          Thread.currentThread().interrupt();
-        }
-        return true;
-      } else {
-        return false;
-      }
-    }
-  }
-
-  public HDFSEventSink() {
-    this(new HDFSWriterFactory());
-  }
-
-  public HDFSEventSink(HDFSWriterFactory writerFactory) {
-    this.writerFactory = writerFactory;
-  }
-
-  @VisibleForTesting
-  Map<String, BucketWriter> getSfWriters() {
-    return sfWriters;
-  }
-
-  // read configuration and setup thresholds
-  @Override
-  public void configure(Context context) {
-    this.context = context;
-
-    filePath = Preconditions.checkNotNull(
-        context.getString("hdfs.path"), "hdfs.path is required");
-    fileName = context.getString("hdfs.filePrefix", defaultFileName);
-    this.suffix = context.getString("hdfs.fileSuffix", defaultSuffix);
-    inUsePrefix = context.getString("hdfs.inUsePrefix", defaultInUsePrefix);
-    boolean emptyInUseSuffix = context.getBoolean("hdfs.emptyInUseSuffix", false);
-    if (emptyInUseSuffix) {
-      inUseSuffix = "";
-      String tmpInUseSuffix = context.getString(IN_USE_SUFFIX_PARAM_NAME);
-      if (tmpInUseSuffix != null) {
-        LOG.warn("Ignoring parameter " + IN_USE_SUFFIX_PARAM_NAME + " for hdfs sink: " + getName());
-      }
-    } else {
-      inUseSuffix = context.getString(IN_USE_SUFFIX_PARAM_NAME, defaultInUseSuffix);
-    }
-    String tzName = context.getString("hdfs.timeZone");
-    timeZone = tzName == null ? null : TimeZone.getTimeZone(tzName);
-    rollInterval = context.getLong("hdfs.rollInterval", defaultRollInterval);
-    rollSize = context.getLong("hdfs.rollSize", defaultRollSize);
-    rollCount = context.getLong("hdfs.rollCount", defaultRollCount);
-    batchSize = context.getLong("hdfs.batchSize", defaultBatchSize);
-    idleTimeout = context.getInteger("hdfs.idleTimeout", 0);
-    String codecName = context.getString("hdfs.codeC");
-    fileType = context.getString("hdfs.fileType", defaultFileType);
-    maxOpenFiles = context.getInteger("hdfs.maxOpenFiles", defaultMaxOpenFiles);
-    callTimeout = context.getLong("hdfs.callTimeout", defaultCallTimeout);
-    threadsPoolSize = context.getInteger("hdfs.threadsPoolSize",
-        defaultThreadPoolSize);
-    rollTimerPoolSize = context.getInteger("hdfs.rollTimerPoolSize",
-        defaultRollTimerPoolSize);
-    String kerbConfPrincipal = context.getString("hdfs.kerberosPrincipal");
-    String kerbKeytab = context.getString("hdfs.kerberosKeytab");
-    String proxyUser = context.getString("hdfs.proxyUser");
-    tryCount = context.getInteger("hdfs.closeTries", defaultTryCount);
-    if (tryCount <= 0) {
-      LOG.warn("Retry count value : " + tryCount + " is not " +
-          "valid. The sink will try to close the file until the file " +
-          "is eventually closed.");
-      tryCount = defaultTryCount;
-    }
-    retryInterval = context.getLong("hdfs.retryInterval", defaultRetryInterval);
-    if (retryInterval <= 0) {
-      LOG.warn("Retry Interval value: " + retryInterval + " is not " +
-          "valid. If the first close of a file fails, " +
-          "it may remain open and will not be renamed.");
-      tryCount = 1;
-    }
-
-    Preconditions.checkArgument(batchSize > 0, "batchSize must be greater than 0");
-    if (codecName == null) {
-      codeC = null;
-      compType = CompressionType.NONE;
-    } else {
-      codeC = getCodec(codecName);
-      // TODO : set proper compression type
-      compType = CompressionType.BLOCK;
-    }
-
-    // Do not allow user to set fileType DataStream with codeC together
-    // To prevent output file with compress extension (like .snappy)
-    if (fileType.equalsIgnoreCase(HDFSWriterFactory.DataStreamType) && codecName != null) {
-      throw new IllegalArgumentException("fileType: " + fileType +
-          " which does NOT support compressed output. Please don't set codeC" +
-          " or change the fileType if compressed output is desired.");
-    }
-
-    if (fileType.equalsIgnoreCase(HDFSWriterFactory.CompStreamType)) {
-      Preconditions.checkNotNull(codeC, "It's essential to set compress codec"
-          + " when fileType is: " + fileType);
-    }
-
-    // get the appropriate executor
-    this.privExecutor = FlumeAuthenticationUtil.getAuthenticator(
-            kerbConfPrincipal, kerbKeytab).proxyAs(proxyUser);
-
-    needRounding = context.getBoolean("hdfs.round", false);
-
-    if (needRounding) {
-      String unit = context.getString("hdfs.roundUnit", "second");
-      if (unit.equalsIgnoreCase("hour")) {
-        this.roundUnit = Calendar.HOUR_OF_DAY;
-      } else if (unit.equalsIgnoreCase("minute")) {
-        this.roundUnit = Calendar.MINUTE;
-      } else if (unit.equalsIgnoreCase("second")) {
-        this.roundUnit = Calendar.SECOND;
-      } else {
-        LOG.warn("Rounding unit is not valid, please set one of" +
-            "minute, hour, or second. Rounding will be disabled");
-        needRounding = false;
-      }
-      this.roundValue = context.getInteger("hdfs.roundValue", 1);
-      if (roundUnit == Calendar.SECOND || roundUnit == Calendar.MINUTE) {
-        Preconditions.checkArgument(roundValue > 0 && roundValue <= 60,
-            "Round value" +
-            "must be > 0 and <= 60");
-      } else if (roundUnit == Calendar.HOUR_OF_DAY) {
-        Preconditions.checkArgument(roundValue > 0 && roundValue <= 24,
-            "Round value" +
-            "must be > 0 and <= 24");
-      }
-    }
-
-    this.useLocalTime = context.getBoolean("hdfs.useLocalTimeStamp", false);
-    if (useLocalTime) {
-      clock = new SystemClock();
-    }
-
-    if (sinkCounter == null) {
-      sinkCounter = new SinkCounter(getName());
-    }
-  }
-
-  private static boolean codecMatches(Class<? extends CompressionCodec> cls, String codecName) {
-    String simpleName = cls.getSimpleName();
-    if (cls.getName().equals(codecName) || simpleName.equalsIgnoreCase(codecName)) {
-      return true;
-    }
-    if (simpleName.endsWith("Codec")) {
-      String prefix = simpleName.substring(0, simpleName.length() - "Codec".length());
-      if (prefix.equalsIgnoreCase(codecName)) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  @VisibleForTesting
-  static CompressionCodec getCodec(String codecName) {
-    Configuration conf = new Configuration();
-    List<Class<? extends CompressionCodec>> codecs = CompressionCodecFactory.getCodecClasses(conf);
-    // Wish we could base this on DefaultCodec but appears not all codec's
-    // extend DefaultCodec(Lzo)
-    CompressionCodec codec = null;
-    ArrayList<String> codecStrs = new ArrayList<String>();
-    codecStrs.add("None");
-    for (Class<? extends CompressionCodec> cls : codecs) {
-      codecStrs.add(cls.getSimpleName());
-      if (codecMatches(cls, codecName)) {
-        try {
-          codec = cls.newInstance();
-        } catch (InstantiationException e) {
-          LOG.error("Unable to instantiate " + cls + " class");
-        } catch (IllegalAccessException e) {
-          LOG.error("Unable to access " + cls + " class");
-        }
-      }
-    }
-
-    if (codec == null) {
-      if (!codecName.equalsIgnoreCase("None")) {
-        throw new IllegalArgumentException("Unsupported compression codec "
-            + codecName + ".  Please choose from: " + codecStrs);
-      }
-    } else if (codec instanceof org.apache.hadoop.conf.Configurable) {
-      // Must check instanceof codec as BZip2Codec doesn't inherit Configurable
-      // Must set the configuration for Configurable objects that may or do use
-      // native libs
-      ((org.apache.hadoop.conf.Configurable) codec).setConf(conf);
-    }
-    return codec;
-  }
-
-
-  /**
-   * Pull events out of channel and send it to HDFS. Take at most batchSize
-   * events per Transaction. Find the corresponding bucket for the event.
-   * Ensure the file is open. Serialize the data and write it to the file on
-   * HDFS. <br/>
-   * This method is not thread safe.
-   */
-  public Status process() throws EventDeliveryException {
-    Channel channel = getChannel();
-    Transaction transaction = channel.getTransaction();
-    transaction.begin();
-    try {
-      Set<BucketWriter> writers = new LinkedHashSet<>();
-      int txnEventCount = 0;
-      for (txnEventCount = 0; txnEventCount < batchSize; txnEventCount++) {
-        Event event = channel.take();
-        if (event == null) {
-          break;
-        }
-
-        // reconstruct the path name by substituting place holders
-        String realPath = BucketPath.escapeString(filePath, event.getHeaders(),
-            timeZone, needRounding, roundUnit, roundValue, useLocalTime);
-        String realName = BucketPath.escapeString(fileName, event.getHeaders(),
-            timeZone, needRounding, roundUnit, roundValue, useLocalTime);
-
-        String lookupPath = realPath + DIRECTORY_DELIMITER + realName;
-        BucketWriter bucketWriter;
-        HDFSWriter hdfsWriter = null;
-        // Callback to remove the reference to the bucket writer from the
-        // sfWriters map so that all buffers used by the HDFS file
-        // handles are garbage collected.
-        WriterCallback closeCallback = new WriterCallback() {
-          @Override
-          public void run(String bucketPath) {
-            LOG.info("Writer callback called.");
-            synchronized (sfWritersLock) {
-              sfWriters.remove(bucketPath);
-            }
-          }
-        };
-        synchronized (sfWritersLock) {
-          bucketWriter = sfWriters.get(lookupPath);
-          // we haven't seen this file yet, so open it and cache the handle
-          if (bucketWriter == null) {
-            hdfsWriter = writerFactory.getWriter(fileType);
-            bucketWriter = initializeBucketWriter(realPath, realName,
-              lookupPath, hdfsWriter, closeCallback);
-            sfWriters.put(lookupPath, bucketWriter);
-          }
-        }
-
-        // Write the data to HDFS
-        try {
-          bucketWriter.append(event);
-        } catch (BucketClosedException ex) {
-          LOG.info("Bucket was closed while trying to append, " +
-                   "reinitializing bucket and writing event.");
-          hdfsWriter = writerFactory.getWriter(fileType);
-          bucketWriter = initializeBucketWriter(realPath, realName,
-            lookupPath, hdfsWriter, closeCallback);
-          synchronized (sfWritersLock) {
-            sfWriters.put(lookupPath, bucketWriter);
-          }
-          bucketWriter.append(event);
-        }
-
-        // track the buckets getting written in this transaction
-        if (!writers.contains(bucketWriter)) {
-          writers.add(bucketWriter);
-        }
-      }
-
-      if (txnEventCount == 0) {
-        sinkCounter.incrementBatchEmptyCount();
-      } else if (txnEventCount == batchSize) {
-        sinkCounter.incrementBatchCompleteCount();
-      } else {
-        sinkCounter.incrementBatchUnderflowCount();
-      }
-
-      // flush all pending buckets before committing the transaction
-      for (BucketWriter bucketWriter : writers) {
-        bucketWriter.flush();
-      }
-
-      transaction.commit();
-
-      if (txnEventCount < 1) {
-        return Status.BACKOFF;
-      } else {
-        sinkCounter.addToEventDrainSuccessCount(txnEventCount);
-        return Status.READY;
-      }
-    } catch (IOException eIO) {
-      transaction.rollback();
-      LOG.warn("HDFS IO error", eIO);
-      sinkCounter.incrementEventWriteFail();
-      return Status.BACKOFF;
-    } catch (Throwable th) {
-      transaction.rollback();
-      LOG.error("process failed", th);
-      sinkCounter.incrementEventWriteOrChannelFail(th);
-      if (th instanceof Error) {
-        throw (Error) th;
-      } else {
-        throw new EventDeliveryException(th);
-      }
-    } finally {
-      transaction.close();
-    }
-  }
-
-  @VisibleForTesting
-  BucketWriter initializeBucketWriter(String realPath,
-      String realName, String lookupPath, HDFSWriter hdfsWriter,
-      WriterCallback closeCallback) {
-    HDFSWriter actualHdfsWriter = mockFs == null ? hdfsWriter : mockWriter;
-    BucketWriter bucketWriter = new BucketWriter(rollInterval,
-        rollSize, rollCount,
-        batchSize, context, realPath, realName, inUsePrefix, inUseSuffix,
-        suffix, codeC, compType, actualHdfsWriter, timedRollerPool,
-        privExecutor, sinkCounter, idleTimeout, closeCallback,
-        lookupPath, callTimeout, callTimeoutPool, retryInterval,
-        tryCount);
-    if (mockFs != null) {
-      bucketWriter.setFileSystem(mockFs);
-    }
-    return bucketWriter;
-  }
-
-  @Override
-  public void stop() {
-    // do not constrain close() calls with a timeout
-    synchronized (sfWritersLock) {
-      for (Entry<String, BucketWriter> entry : sfWriters.entrySet()) {
-        LOG.info("Closing {}", entry.getKey());
-
-        try {
-          entry.getValue().close(false, true);
-        } catch (Exception ex) {
-          LOG.warn("Exception while closing " + entry.getKey() + ". " +
-                  "Exception follows.", ex);
-          if (ex instanceof InterruptedException) {
-            Thread.currentThread().interrupt();
-          }
-        }
-      }
-    }
-
-    // shut down all our thread pools
-    ExecutorService[] toShutdown = { callTimeoutPool, timedRollerPool };
-    for (ExecutorService execService : toShutdown) {
-      execService.shutdown();
-      try {
-        while (execService.isTerminated() == false) {
-          execService.awaitTermination(
-                  Math.max(defaultCallTimeout, callTimeout), TimeUnit.MILLISECONDS);
-        }
-      } catch (InterruptedException ex) {
-        LOG.warn("shutdown interrupted on " + execService, ex);
-      }
-    }
-
-    callTimeoutPool = null;
-    timedRollerPool = null;
-
-    synchronized (sfWritersLock) {
-      sfWriters.clear();
-      sfWriters = null;
-    }
-    sinkCounter.stop();
-    super.stop();
-  }
-
-  @Override
-  public void start() {
-    String timeoutName = "hdfs-" + getName() + "-call-runner-%d";
-    callTimeoutPool = Executors.newFixedThreadPool(threadsPoolSize,
-            new ThreadFactoryBuilder().setNameFormat(timeoutName).build());
-
-    String rollerName = "hdfs-" + getName() + "-roll-timer-%d";
-    timedRollerPool = Executors.newScheduledThreadPool(rollTimerPoolSize,
-            new ThreadFactoryBuilder().setNameFormat(rollerName).build());
-
-    this.sfWriters = new WriterLinkedHashMap(maxOpenFiles);
-    sinkCounter.start();
-    super.start();
-  }
-
-  @Override
-  public String toString() {
-    return "{ Sink type:" + getClass().getSimpleName() + ", name:" + getName() +
-            " }";
-  }
-
-  @VisibleForTesting
-  void setBucketClock(Clock clock) {
-    BucketPath.setClock(clock);
-  }
-
-  @VisibleForTesting
-  void setMockFs(FileSystem mockFs) {
-    this.mockFs = mockFs;
-  }
-
-  @VisibleForTesting
-  void setMockWriter(HDFSWriter writer) {
-    this.mockWriter = writer;
-  }
-
-  @VisibleForTesting
-  int getTryCount() {
-    return tryCount;
-  }
-
-  @Override
-  public long getBatchSize() {
-    return batchSize;
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSSequenceFile.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSSequenceFile.java
deleted file mode 100644
index c5430ba65..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSSequenceFile.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HDFSSequenceFile extends AbstractHDFSWriter {
-
-  private static final Logger logger =
-      LoggerFactory.getLogger(HDFSSequenceFile.class);
-  private SequenceFile.Writer writer;
-  private String writeFormat;
-  private Context serializerContext;
-  private SequenceFileSerializer serializer;
-  private boolean useRawLocalFileSystem;
-  private FSDataOutputStream outStream = null;
-
-  public HDFSSequenceFile() {
-    writer = null;
-  }
-
-  @Override
-  public void configure(Context context) {
-    super.configure(context);
-
-    // use binary writable serialize by default
-    writeFormat = context.getString("hdfs.writeFormat",
-      SequenceFileSerializerType.Writable.name());
-    useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
-        false);
-    serializerContext = new Context(
-            context.getSubProperties(SequenceFileSerializerFactory.CTX_PREFIX));
-    serializer = SequenceFileSerializerFactory
-            .getSerializer(writeFormat, serializerContext);
-    logger.info("writeFormat = " + writeFormat + ", UseRawLocalFileSystem = "
-        + useRawLocalFileSystem);
-  }
-
-  @Override
-  public void open(String filePath) throws IOException {
-    open(filePath, null, CompressionType.NONE);
-  }
-
-  @Override
-  public void open(String filePath, CompressionCodec codeC,
-      CompressionType compType) throws IOException {
-    Configuration conf = new Configuration();
-    Path dstPath = new Path(filePath);
-    FileSystem hdfs = dstPath.getFileSystem(conf);
-    open(dstPath, codeC, compType, conf, hdfs);
-  }
-
-  protected void open(Path dstPath, CompressionCodec codeC,
-      CompressionType compType, Configuration conf, FileSystem hdfs)
-          throws IOException {
-    if (useRawLocalFileSystem) {
-      if (hdfs instanceof LocalFileSystem) {
-        hdfs = ((LocalFileSystem)hdfs).getRaw();
-      } else {
-        logger.warn("useRawLocalFileSystem is set to true but file system " +
-            "is not of type LocalFileSystem: " + hdfs.getClass().getName());
-      }
-    }
-    if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) {
-      outStream = hdfs.append(dstPath);
-    } else {
-      outStream = hdfs.create(dstPath);
-    }
-    writer = SequenceFile.createWriter(conf, outStream,
-        serializer.getKeyClass(), serializer.getValueClass(), compType, codeC);
-
-    registerCurrentStream(outStream, hdfs, dstPath);
-  }
-
-  @Override
-  public void append(Event e) throws IOException {
-    for (SequenceFileSerializer.Record record : serializer.serialize(e)) {
-      writer.append(record.getKey(), record.getValue());
-    }
-  }
-
-  @Override
-  public void sync() throws IOException {
-    writer.sync();
-    hflushOrSync(outStream);
-  }
-
-  @Override
-  public void close() throws IOException {
-    writer.close();
-    outStream.close();
-    unregisterCurrentStream();
-  }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextSerializer.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextSerializer.java
deleted file mode 100644
index 32fd20662..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextSerializer.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.util.Collections;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.LongWritable;
-
-public class HDFSTextSerializer implements SequenceFileSerializer {
-
-  private Text makeText(Event e) {
-    Text textObject = new Text();
-    textObject.set(e.getBody(), 0, e.getBody().length);
-    return textObject;
-  }
-
-  @Override
-  public Class<LongWritable> getKeyClass() {
-    return LongWritable.class;
-  }
-
-  @Override
-  public Class<Text> getValueClass() {
-    return Text.class;
-  }
-
-  @Override
-  public Iterable<Record> serialize(Event e) {
-    Object key = getKey(e);
-    Object value = getValue(e);
-    return Collections.singletonList(new Record(key, value));
-  }
-
-  private Object getKey(Event e) {
-    // Write the data to HDFS
-    String timestamp = e.getHeaders().get("timestamp");
-    long eventStamp;
-
-    if (timestamp == null) {
-      eventStamp = System.currentTimeMillis();
-    } else {
-      eventStamp = Long.valueOf(timestamp);
-    }
-    return new LongWritable(eventStamp);
-  }
-
-  private Object getValue(Event e) {
-    return makeText(e);
-  }
-
-  public static class Builder implements SequenceFileSerializer.Builder {
-
-    @Override
-    public SequenceFileSerializer build(Context context) {
-      return new HDFSTextSerializer();
-    }
-
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableSerializer.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableSerializer.java
deleted file mode 100644
index b25a6ea50..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableSerializer.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.LongWritable;
-
-import java.util.Collections;
-
-public class HDFSWritableSerializer implements SequenceFileSerializer {
-
-  private BytesWritable makeByteWritable(Event e) {
-    BytesWritable bytesObject = new BytesWritable();
-    bytesObject.set(e.getBody(), 0, e.getBody().length);
-    return bytesObject;
-  }
-
-  @Override
-  public Class<LongWritable> getKeyClass() {
-    return LongWritable.class;
-  }
-
-  @Override
-  public Class<BytesWritable> getValueClass() {
-    return BytesWritable.class;
-  }
-
-  @Override
-  public Iterable<Record> serialize(Event e) {
-    Object key = getKey(e);
-    Object value = getValue(e);
-    return Collections.singletonList(new Record(key, value));
-  }
-
-  private Object getKey(Event e) {
-    String timestamp = e.getHeaders().get("timestamp");
-    long eventStamp;
-
-    if (timestamp == null) {
-      eventStamp = System.currentTimeMillis();
-    } else {
-      eventStamp = Long.valueOf(timestamp);
-    }
-    return new LongWritable(eventStamp);
-  }
-
-  private Object getValue(Event e) {
-    return makeByteWritable(e);
-  }
-
-  public static class Builder implements SequenceFileSerializer.Builder {
-
-    @Override
-    public SequenceFileSerializer build(Context context) {
-      return new HDFSWritableSerializer();
-    }
-
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriter.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriter.java
deleted file mode 100644
index 44a984ac4..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-
-import org.apache.flume.Event;
-import org.apache.flume.annotations.InterfaceAudience;
-import org.apache.flume.annotations.InterfaceStability;
-import org.apache.flume.conf.Configurable;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface HDFSWriter extends Configurable {
-
-  public void open(String filePath) throws IOException;
-
-  public void open(String filePath, CompressionCodec codec,
-      CompressionType cType) throws IOException;
-
-  public void append(Event e) throws IOException;
-
-  public void sync() throws IOException;
-
-  public void close() throws IOException;
-
-  public boolean isUnderReplicated();
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriterFactory.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriterFactory.java
deleted file mode 100644
index a90d53643..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriterFactory.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-
-public class HDFSWriterFactory {
-  static final String SequenceFileType = "SequenceFile";
-  static final String DataStreamType = "DataStream";
-  static final String CompStreamType = "CompressedStream";
-
-  public HDFSWriterFactory() {
-
-  }
-
-  public HDFSWriter getWriter(String fileType) throws IOException {
-    if (fileType.equalsIgnoreCase(SequenceFileType)) {
-      return new HDFSSequenceFile();
-    } else if (fileType.equalsIgnoreCase(DataStreamType)) {
-      return new HDFSDataStream();
-    } else if (fileType.equalsIgnoreCase(CompStreamType)) {
-      return new HDFSCompressedDataStream();
-    } else {
-      throw new IOException("File type " + fileType + " not supported");
-    }
-  }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/KerberosUser.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/KerberosUser.java
deleted file mode 100644
index 43297e25b..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/KerberosUser.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.flume.sink.hdfs;
-
-/**
- * Simple Pair class used to define a unique (principal, keyTab) combination.
- */
-public class KerberosUser {
-
-  private final String principal;
-  private final String keyTab;
-
-  public KerberosUser(String principal, String keyTab) {
-    this.principal = principal;
-    this.keyTab = keyTab;
-  }
-
-  public String getPrincipal() {
-    return principal;
-  }
-
-  public String getKeyTab() {
-    return keyTab;
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (obj == null) {
-      return false;
-    }
-    if (getClass() != obj.getClass()) {
-      return false;
-    }
-    final KerberosUser other = (KerberosUser) obj;
-    if ((this.principal == null) ?
-        (other.principal != null) :
-        !this.principal.equals(other.principal)) {
-      return false;
-    }
-    if ((this.keyTab == null) ? (other.keyTab != null) : !this.keyTab.equals(other.keyTab)) {
-      return false;
-    }
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    int hash = 7;
-    hash = 41 * hash + (this.principal != null ? this.principal.hashCode() : 0);
-    hash = 41 * hash + (this.keyTab != null ? this.keyTab.hashCode() : 0);
-    return hash;
-  }
-
-  @Override
-  public String toString() {
-    return "{ principal: " + principal + ", keytab: " + keyTab + " }";
-  }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializer.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializer.java
deleted file mode 100644
index ec2b7601d..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializer.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-
-public interface SequenceFileSerializer {
-
-  Class<?> getKeyClass();
-
-  Class<?> getValueClass();
-
-  /**
-   * Format the given event into zero, one or more SequenceFile records
-   *
-   * @param e
-   *         event
-   * @return a list of records corresponding to the given event
-   */
-  Iterable<Record> serialize(Event e);
-
-  /**
-   * Knows how to construct this output formatter.<br/>
-   * <b>Note: Implementations MUST provide a public a no-arg constructor.</b>
-   */
-  public interface Builder {
-    public SequenceFileSerializer build(Context context);
-  }
-
-  /**
-   * A key-value pair making up a record in an HDFS SequenceFile
-   */
-  public static class Record {
-    private final Object key;
-    private final Object value;
-
-    public Record(Object key, Object value) {
-      this.key = key;
-      this.value = value;
-    }
-
-    public Object getKey() {
-      return key;
-    }
-
-    public Object getValue() {
-      return value;
-    }
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerFactory.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerFactory.java
deleted file mode 100644
index 56788364f..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerFactory.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import com.google.common.base.Preconditions;
-import org.apache.flume.Context;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class SequenceFileSerializerFactory {
-
-  private static final Logger logger =
-      LoggerFactory.getLogger(SequenceFileSerializerFactory.class);
-
-  /**
-   * {@link Context} prefix
-   */
-  static final String CTX_PREFIX = "writeFormat.";
-
-  @SuppressWarnings("unchecked")
-  static SequenceFileSerializer getSerializer(String formatType,
-                                              Context context) {
-
-    Preconditions.checkNotNull(formatType,
-        "serialize type must not be null");
-
-    // try to find builder class in enum of known formatters
-    SequenceFileSerializerType type;
-    try {
-      type = SequenceFileSerializerType.valueOf(formatType);
-    } catch (IllegalArgumentException e) {
-      logger.debug("Not in enum, loading builder class: {}", formatType);
-      type = SequenceFileSerializerType.Other;
-    }
-    Class<? extends SequenceFileSerializer.Builder> builderClass =
-        type.getBuilderClass();
-
-    // handle the case where they have specified their own builder in the config
-    if (builderClass == null) {
-      try {
-        Class c = Class.forName(formatType);
-        if (c != null && SequenceFileSerializer.Builder.class.isAssignableFrom(c)) {
-          builderClass = (Class<? extends SequenceFileSerializer.Builder>) c;
-        } else {
-          logger.error("Unable to instantiate Builder from {}", formatType);
-          return null;
-        }
-      } catch (ClassNotFoundException ex) {
-        logger.error("Class not found: " + formatType, ex);
-        return null;
-      } catch (ClassCastException ex) {
-        logger.error("Class does not extend " +
-            SequenceFileSerializer.Builder.class.getCanonicalName() + ": " +
-            formatType, ex);
-        return null;
-      }
-    }
-
-    // build the builder
-    SequenceFileSerializer.Builder builder;
-    try {
-      builder = builderClass.newInstance();
-    } catch (InstantiationException ex) {
-      logger.error("Cannot instantiate builder: " + formatType, ex);
-      return null;
-    } catch (IllegalAccessException ex) {
-      logger.error("Cannot instantiate builder: " + formatType, ex);
-      return null;
-    }
-
-    return builder.build(context);
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerType.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerType.java
deleted file mode 100644
index 2ad7689f2..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerType.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-public enum SequenceFileSerializerType {
-  Writable(HDFSWritableSerializer.Builder.class),
-  Text(HDFSTextSerializer.Builder.class),
-  Other(null);
-
-  private final Class<? extends SequenceFileSerializer.Builder> builderClass;
-
-  SequenceFileSerializerType(Class<? extends SequenceFileSerializer.Builder> builderClass) {
-    this.builderClass = builderClass;
-  }
-
-  public Class<? extends SequenceFileSerializer.Builder> getBuilderClass() {
-    return builderClass;
-  }
-
-}
-
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSBadDataStream.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSBadDataStream.java
deleted file mode 100644
index d325233ac..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSBadDataStream.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-import org.apache.flume.Event;
-
-public class HDFSBadDataStream extends HDFSDataStream {
-  public class HDFSBadSeqWriter extends HDFSSequenceFile {
-    @Override
-    public void append(Event e) throws IOException {
-
-      if (e.getHeaders().containsKey("fault")) {
-        throw new IOException("Injected fault");
-      } else if (e.getHeaders().containsKey("slow")) {
-        long waitTime = Long.parseLong(e.getHeaders().get("slow"));
-        try {
-          Thread.sleep(waitTime);
-        } catch (InterruptedException eT) {
-          throw new IOException("append interrupted", eT);
-        }
-      }
-      super.append(e);
-    }
-
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestSeqWriter.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestSeqWriter.java
deleted file mode 100644
index f1dadf1f2..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestSeqWriter.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import org.apache.flume.Event;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-
-import java.io.IOException;
-
-public class HDFSTestSeqWriter extends HDFSSequenceFile {
-  protected volatile boolean closed;
-  protected volatile boolean opened;
-
-  private int openCount = 0;
-
-  HDFSTestSeqWriter(int openCount) {
-    this.openCount = openCount;
-  }
-
-  @Override
-  public void open(String filePath, CompressionCodec codeC, CompressionType compType)
-      throws IOException {
-    super.open(filePath, codeC, compType);
-    if (closed) {
-      opened = true;
-    }
-  }
-
-  @Override
-  public void append(Event e) throws IOException {
-
-    if (e.getHeaders().containsKey("fault")) {
-      throw new IOException("Injected fault");
-    } else if (e.getHeaders().containsKey("fault-once")) {
-      e.getHeaders().remove("fault-once");
-      throw new IOException("Injected fault");
-    } else if (e.getHeaders().containsKey("fault-until-reopen")) {
-      // opening first time.
-      if (openCount == 1) {
-        throw new IOException("Injected fault-until-reopen");
-      }
-    } else if (e.getHeaders().containsKey("slow")) {
-      long waitTime = Long.parseLong(e.getHeaders().get("slow"));
-      try {
-        Thread.sleep(waitTime);
-      } catch (InterruptedException eT) {
-        throw new IOException("append interrupted", eT);
-      }
-    }
-
-    super.append(e);
-  }
-
-  @Override
-  public void close() throws IOException {
-    closed = true;
-    super.close();
-  }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestWriterFactory.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestWriterFactory.java
deleted file mode 100644
index 70bd9e643..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestWriterFactory.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-public class HDFSTestWriterFactory extends HDFSWriterFactory {
-  static final String TestSequenceFileType = "SequenceFile";
-  static final String BadDataStreamType = "DataStream";
-
-  // so we can get a handle to this one in our test.
-  AtomicInteger openCount = new AtomicInteger(0);
-
-  @Override
-  public HDFSWriter getWriter(String fileType) throws IOException {
-    if (fileType == TestSequenceFileType) {
-      return new HDFSTestSeqWriter(openCount.incrementAndGet());
-    } else if (fileType == BadDataStreamType) {
-      return new HDFSBadDataStream();
-    } else {
-      throw new IOException("File type " + fileType + " not supported");
-    }
-  }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockDataStream.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockDataStream.java
deleted file mode 100644
index a85a99f1c..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockDataStream.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-
-class MockDataStream extends HDFSDataStream {
-  private final FileSystem fs;
-
-  MockDataStream(FileSystem fs) {
-    this.fs = fs;
-  }
-
-  @Override
-  protected FileSystem getDfs(Configuration conf, Path dstPath) throws IOException {
-    return fs;
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFileSystem.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFileSystem.java
deleted file mode 100644
index a079b8316..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFileSystem.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-import java.net.URI;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.util.Progressable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MockFileSystem extends FileSystem {
-
-  private static final Logger logger =
-      LoggerFactory.getLogger(MockFileSystem.class);
-
-  FileSystem fs;
-  int numberOfRetriesRequired;
-  MockFsDataOutputStream latestOutputStream;
-  int currentRenameAttempts;
-  boolean closeSucceed = true;
-
-  public MockFileSystem(FileSystem fs, int numberOfRetriesRequired) {
-    this.fs = fs;
-    this.numberOfRetriesRequired = numberOfRetriesRequired;
-  }
-
-  public MockFileSystem(FileSystem fs,
-                        int numberOfRetriesRequired, boolean closeSucceed) {
-    this.fs = fs;
-    this.numberOfRetriesRequired = numberOfRetriesRequired;
-    this.closeSucceed = closeSucceed;
-  }
-
-  @Override
-  public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2)
-      throws IOException {
-
-    latestOutputStream = new MockFsDataOutputStream(
-      fs.append(arg0, arg1, arg2), closeSucceed);
-
-    return latestOutputStream;
-  }
-
-  @Override
-  public FSDataOutputStream create(Path arg0) throws IOException {
-    latestOutputStream = new MockFsDataOutputStream(fs.create(arg0), closeSucceed);
-    return latestOutputStream;
-  }
-
-  @Override
-  public FSDataOutputStream create(Path arg0, FsPermission arg1, boolean arg2, int arg3,
-                                   short arg4, long arg5, Progressable arg6)
-      throws IOException {
-    throw new IOException("Not a real file system");
-  }
-
-  @Override
-  @Deprecated
-  public boolean delete(Path arg0) throws IOException {
-    return fs.delete(arg0);
-  }
-
-  @Override
-  public boolean delete(Path arg0, boolean arg1) throws IOException {
-    return fs.delete(arg0, arg1);
-  }
-
-  @Override
-  public FileStatus getFileStatus(Path arg0) throws IOException {
-    return fs.getFileStatus(arg0);
-  }
-
-  @Override
-  public URI getUri() {
-    return fs.getUri();
-  }
-
-  @Override
-  public Path getWorkingDirectory() {
-    return fs.getWorkingDirectory();
-  }
-
-  @Override
-  public FileStatus[] listStatus(Path arg0) throws IOException {
-    return fs.listStatus(arg0);
-  }
-
-  @Override
-  public boolean mkdirs(Path arg0, FsPermission arg1) throws IOException {
-    // TODO Auto-generated method stub
-    return fs.mkdirs(arg0, arg1);
-  }
-
-  @Override
-  public FSDataInputStream open(Path arg0, int arg1) throws IOException {
-    return fs.open(arg0, arg1);
-  }
-
-  @Override
-  public boolean rename(Path arg0, Path arg1) throws IOException {
-    currentRenameAttempts++;
-    logger.info("Attempting to Rename: '" + currentRenameAttempts + "' of '" +
-                numberOfRetriesRequired + "'");
-    if (currentRenameAttempts >= numberOfRetriesRequired || numberOfRetriesRequired == 0) {
-      logger.info("Renaming file");
-      return fs.rename(arg0, arg1);
-    } else {
-      throw new IOException("MockIOException");
-    }
-  }
-
-  @Override
-  public void setWorkingDirectory(Path arg0) {
-    fs.setWorkingDirectory(arg0);
-  }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFsDataOutputStream.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFsDataOutputStream.java
deleted file mode 100644
index f5d579c70..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFsDataOutputStream.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
-+ * Licensed to the Apache Software Foundation (ASF) under one
-+ * or more contributor license agreements.  See the NOTICE file
-+ * distributed with this work for additional information
-+ * regarding copyright ownership.  The ASF licenses this file
-+ * to you under the Apache License, Version 2.0 (the
-+ * "License"); you may not use this file except in compliance
-+ * with the License.  You may obtain a copy of the License at
-+ *
-+ *     http://www.apache.org/licenses/LICENSE-2.0
-+ *
-+ * Unless required by applicable law or agreed to in writing, software
-+ * distributed under the License is distributed on an "AS IS" BASIS,
-+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+ * See the License for the specific language governing permissions and
-+ * limitations under the License.
-+ */
-package org.apache.flume.sink.hdfs;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-public class MockFsDataOutputStream extends FSDataOutputStream {
-
-  private static final Logger logger =
-      LoggerFactory.getLogger(MockFsDataOutputStream.class);
-
-  boolean closeSucceed;
-
-  public MockFsDataOutputStream(FSDataOutputStream wrapMe, boolean closeSucceed)
-      throws IOException {
-    super(wrapMe.getWrappedStream(), null);
-    this.closeSucceed = closeSucceed;
-  }
-
-  @Override
-  public void close() throws IOException {
-    logger.info("Close Succeeded - " + closeSucceed);
-    if (closeSucceed) {
-      logger.info("closing file");
-      super.close();
-    } else {
-      throw new IOException("MockIOException");
-    }
-  }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockHDFSWriter.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockHDFSWriter.java
deleted file mode 100644
index c57496bb1..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockHDFSWriter.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MockHDFSWriter implements HDFSWriter {
-
-  private int filesOpened = 0;
-  private int filesClosed = 0;
-  private int bytesWritten = 0;
-  private int eventsWritten = 0;
-  private String filePath = null;
-  private static final Logger logger = LoggerFactory.getLogger(MockHDFSWriter.class);
-  private int numberOfRetriesRequired;
-  public volatile AtomicInteger currentCloseAttempts = new AtomicInteger(0);
-
-  public MockHDFSWriter(int numberOfRetriesRequired) {
-    this.numberOfRetriesRequired = numberOfRetriesRequired;
-  }
-
-  public MockHDFSWriter() {
-    this.numberOfRetriesRequired = 0;
-  }
-
-  public int getFilesOpened() {
-    return filesOpened;
-  }
-
-  public int getFilesClosed() {
-    return filesClosed;
-  }
-
-  public int getBytesWritten() {
-    return bytesWritten;
-  }
-
-  public int getEventsWritten() {
-    return eventsWritten;
-  }
-
-  public String getOpenedFilePath() {
-    return filePath;
-  }
-
-  public void configure(Context context) {
-    // no-op
-  }
-
-  public void open(String filePath) throws IOException {
-    this.filePath = filePath;
-    filesOpened++;
-  }
-
-  public void open(String filePath, CompressionCodec codec, CompressionType cType)
-      throws IOException {
-    this.filePath = filePath;
-    filesOpened++;
-  }
-
-  public void append(Event e) throws IOException {
-    eventsWritten++;
-    bytesWritten += e.getBody().length;
-  }
-
-  public void sync() throws IOException {
-    // does nothing
-  }
-
-  public void close() throws IOException {
-    filesClosed++;
-    int curr = currentCloseAttempts.incrementAndGet();
-    logger.info("Attempting to close: '" + currentCloseAttempts + "' of '" +
-        numberOfRetriesRequired + "'");
-    if (curr >= numberOfRetriesRequired || numberOfRetriesRequired == 0) {
-      logger.info("closing file");
-    } else {
-      throw new IOException("MockIOException");
-    }
-  }
-
-  @Override
-  public boolean isUnderReplicated() {
-    return false;
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomSerializer.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomSerializer.java
deleted file mode 100644
index 72164fdcf..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomSerializer.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.LongWritable;
-
-import java.util.Arrays;
-
-public class MyCustomSerializer implements SequenceFileSerializer {
-
-  @Override
-  public Class<LongWritable> getKeyClass() {
-    return LongWritable.class;
-  }
-
-  @Override
-  public Class<BytesWritable> getValueClass() {
-    return BytesWritable.class;
-  }
-
-  @Override
-  public Iterable<Record> serialize(Event e) {
-    return Arrays.asList(
-        new Record(new LongWritable(1234L), new BytesWritable(new byte[10])),
-        new Record(new LongWritable(4567L), new BytesWritable(new byte[20]))
-    );
-  }
-
-  public static class Builder implements SequenceFileSerializer.Builder {
-
-    @Override
-    public SequenceFileSerializer build(Context context) {
-      return new MyCustomSerializer();
-    }
-
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestAvroEventSerializer.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestAvroEventSerializer.java
deleted file mode 100644
index 6b38da2a0..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestAvroEventSerializer.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Arrays;
-import org.apache.avro.Schema;
-import org.apache.avro.file.DataFileReader;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.generic.GenericRecordBuilder;
-import org.apache.avro.io.BinaryEncoder;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.io.EncoderFactory;
-import org.apache.avro.reflect.ReflectDatumWriter;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.event.EventBuilder;
-import org.apache.flume.serialization.AvroEventSerializerConfigurationConstants;
-import org.apache.flume.serialization.EventSerializer;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.After;
-
-public class TestAvroEventSerializer {
-
-  private File file;
-
-  @Before
-  public void setUp() throws Exception {
-    file = File.createTempFile(getClass().getSimpleName(), "");
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    file.delete();
-  }
-
-  @Test
-  public void testNoCompression() throws IOException {
-    createAvroFile(file, null, false, false);
-    validateAvroFile(file);
-  }
-
-  @Test
-  public void testNullCompression() throws IOException {
-    createAvroFile(file, "null", false, false);
-    validateAvroFile(file);
-  }
-
-  @Test
-  public void testDeflateCompression() throws IOException {
-    createAvroFile(file, "deflate", false, false);
-    validateAvroFile(file);
-  }
-
-  @Test
-  public void testSnappyCompression() throws IOException {
-    createAvroFile(file, "snappy", false, false);
-    validateAvroFile(file);
-  }
-
-  @Test
-  public void testSchemaUrl() throws IOException {
-    createAvroFile(file, null, true, false);
-    validateAvroFile(file);
-  }
-
-  @Test
-  public void testStaticSchemaUrl() throws IOException {
-    createAvroFile(file,null,false, true);
-    validateAvroFile(file);
-  }
-
-  @Test
-  public void testBothUrls() throws IOException {
-    createAvroFile(file,null,true,true);
-    validateAvroFile(file);
-  }
-
-  public void createAvroFile(File file, String codec, boolean useSchemaUrl,
-                             boolean useStaticSchemaUrl) throws IOException {
-    // serialize a few events using the reflection-based avro serializer
-    OutputStream out = new FileOutputStream(file);
-
-    Context ctx = new Context();
-    if (codec != null) {
-      ctx.put("compressionCodec", codec);
-    }
-
-    Schema schema = Schema.createRecord("myrecord", null, null, false);
-    schema.setFields(Arrays.asList(new Schema.Field[]{
-        new Schema.Field("message", Schema.create(Schema.Type.STRING), null, null)
-    }));
-    GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
-    File schemaFile = null;
-    if (useSchemaUrl || useStaticSchemaUrl) {
-      schemaFile = File.createTempFile(getClass().getSimpleName(), ".avsc");
-      Files.write(schema.toString(), schemaFile, Charsets.UTF_8);
-    }
-
-    if (useStaticSchemaUrl) {
-      ctx.put(AvroEventSerializerConfigurationConstants.STATIC_SCHEMA_URL,
-              schemaFile.toURI().toURL().toExternalForm());
-    }
-
-    EventSerializer.Builder builder = new AvroEventSerializer.Builder();
-    EventSerializer serializer = builder.build(ctx, out);
-
-    serializer.afterCreate();
-    for (int i = 0; i < 3; i++) {
-      GenericRecord record = recordBuilder.set("message", "Hello " + i).build();
-      Event event = EventBuilder.withBody(serializeAvro(record, schema));
-      if (schemaFile == null && !useSchemaUrl) {
-        event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_LITERAL_HEADER,
-            schema.toString());
-      } else if (useSchemaUrl) {
-        event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_URL_HEADER,
-            schemaFile.toURI().toURL().toExternalForm());
-      }
-      serializer.write(event);
-    }
-    serializer.flush();
-    serializer.beforeClose();
-    out.flush();
-    out.close();
-    if (schemaFile != null ) {
-      schemaFile.delete();
-    }
-
-  }
-
-  private byte[] serializeAvro(Object datum, Schema schema) throws IOException {
-    ByteArrayOutputStream out = new ByteArrayOutputStream();
-    ReflectDatumWriter<Object> writer = new ReflectDatumWriter<Object>(schema);
-    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
-    out.reset();
-    writer.write(datum, encoder);
-    encoder.flush();
-    return out.toByteArray();
-  }
-
-  public void validateAvroFile(File file) throws IOException {
-    // read the events back using GenericRecord
-    DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
-    DataFileReader<GenericRecord> fileReader =
-        new DataFileReader<GenericRecord>(file, reader);
-    GenericRecord record = new GenericData.Record(fileReader.getSchema());
-    int numEvents = 0;
-    while (fileReader.hasNext()) {
-      fileReader.next(record);
-      String bodyStr = record.get("message").toString();
-      System.out.println(bodyStr);
-      numEvents++;
-    }
-    fileReader.close();
-    Assert.assertEquals("Should have found a total of 3 events", 3, numEvents);
-  }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestBucketWriter.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestBucketWriter.java
deleted file mode 100644
index 64a89e171..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestBucketWriter.java
+++ /dev/null
@@ -1,719 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import com.google.common.base.Charsets;
-import org.apache.flume.Clock;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.SystemClock;
-import org.apache.flume.auth.FlumeAuthenticationUtil;
-import org.apache.flume.auth.PrivilegedExecutor;
-import org.apache.flume.event.EventBuilder;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.flume.sink.hdfs.HDFSEventSink.WriterCallback;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Calendar;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class TestBucketWriter {
-
-  private static Logger logger = LoggerFactory.getLogger(TestBucketWriter.class);
-  private Context ctx = new Context();
-
-  private static ScheduledExecutorService timedRollerPool;
-  private static PrivilegedExecutor proxy;
-
-  @BeforeClass
-  public static void setup() {
-    timedRollerPool = Executors.newSingleThreadScheduledExecutor();
-    proxy = FlumeAuthenticationUtil.getAuthenticator(null, null).proxyAs(null);
-  }
-
-  @AfterClass
-  public static void teardown() throws InterruptedException {
-    timedRollerPool.shutdown();
-    timedRollerPool.awaitTermination(2, TimeUnit.SECONDS);
-    timedRollerPool.shutdownNow();
-  }
-
-  @Test
-  public void testEventCountingRoller() throws IOException, InterruptedException {
-    int maxEvents = 100;
-    MockHDFSWriter hdfsWriter = new MockHDFSWriter();
-    BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
-        .setRollCount(maxEvents).build();
-
-    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
-    for (int i = 0; i < 1000; i++) {
-      bucketWriter.append(e);
-    }
-
-    logger.info("Number of events written: {}", hdfsWriter.getEventsWritten());
-    logger.info("Number of bytes written: {}", hdfsWriter.getBytesWritten());
-    logger.info("Number of files opened: {}", hdfsWriter.getFilesOpened());
-
-    Assert.assertEquals("events written", 1000, hdfsWriter.getEventsWritten());
-    Assert.assertEquals("bytes written", 3000, hdfsWriter.getBytesWritten());
-    Assert.assertEquals("files opened", 10, hdfsWriter.getFilesOpened());
-  }
-
-  @Test
-  public void testSizeRoller() throws IOException, InterruptedException {
-    int maxBytes = 300;
-    MockHDFSWriter hdfsWriter = new MockHDFSWriter();
-    BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
-        .setRollSize(maxBytes).build();
-
-    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
-    for (int i = 0; i < 1000; i++) {
-      bucketWriter.append(e);
-    }
-
-    logger.info("Number of events written: {}", hdfsWriter.getEventsWritten());
-    logger.info("Number of bytes written: {}", hdfsWriter.getBytesWritten());
-    logger.info("Number of files opened: {}", hdfsWriter.getFilesOpened());
-
-    Assert.assertEquals("events written", 1000, hdfsWriter.getEventsWritten());
-    Assert.assertEquals("bytes written", 3000, hdfsWriter.getBytesWritten());
-    Assert.assertEquals("files opened", 10, hdfsWriter.getFilesOpened());
-  }
-
-  @Test
-  public void testIntervalRoller() throws IOException, InterruptedException {
-    final int ROLL_INTERVAL = 1; // seconds
-    final int NUM_EVENTS = 10;
-    final AtomicBoolean calledBack = new AtomicBoolean(false);
-
-    MockHDFSWriter hdfsWriter = new MockHDFSWriter();
-    BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
-        .setRollInterval(ROLL_INTERVAL)
-        .setOnCloseCallback(new HDFSEventSink.WriterCallback() {
-          @Override
-          public void run(String filePath) {
-            calledBack.set(true);
-          }
-        }).build();
-
-    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
-    long startNanos = System.nanoTime();
-    for (int i = 0; i < NUM_EVENTS - 1; i++) {
-      bucketWriter.append(e);
-    }
-
-    // sleep to force a roll... wait 2x interval just to be sure
-    Thread.sleep(2 * ROLL_INTERVAL * 1000L);
-
-    Assert.assertTrue(bucketWriter.closed.get());
-    Assert.assertTrue(calledBack.get());
-
-    bucketWriter = new BucketWriterBuilder(hdfsWriter)
-        .setRollInterval(ROLL_INTERVAL).build();
-    // write one more event (to reopen a new file so we will roll again later)
-    bucketWriter.append(e);
-
-    long elapsedMillis = TimeUnit.MILLISECONDS.convert(
-        System.nanoTime() - startNanos, TimeUnit.NANOSECONDS);
-    long elapsedSeconds = elapsedMillis / 1000L;
-
-    logger.info("Time elapsed: {} milliseconds", elapsedMillis);
-    logger.info("Number of events written: {}", hdfsWriter.getEventsWritten());
-    logger.info("Number of bytes written: {}", hdfsWriter.getBytesWritten());
-    logger.info("Number of files opened: {}", hdfsWriter.getFilesOpened());
-    logger.info("Number of files closed: {}", hdfsWriter.getFilesClosed());
-
-    Assert.assertEquals("events written", NUM_EVENTS,
-        hdfsWriter.getEventsWritten());
-    Assert.assertEquals("bytes written", e.getBody().length * NUM_EVENTS,
-        hdfsWriter.getBytesWritten());
-    Assert.assertEquals("files opened", 2, hdfsWriter.getFilesOpened());
-
-    // before auto-roll
-    Assert.assertEquals("files closed", 1, hdfsWriter.getFilesClosed());
-
-    logger.info("Waiting for roll...");
-    Thread.sleep(2 * ROLL_INTERVAL * 1000L);
-
-    logger.info("Number of files closed: {}", hdfsWriter.getFilesClosed());
-    Assert.assertEquals("files closed", 2, hdfsWriter.getFilesClosed());
-  }
-
-  @Test
-  public void testIntervalRollerBug() throws IOException, InterruptedException {
-    final int ROLL_INTERVAL = 1; // seconds
-    final int NUM_EVENTS = 10;
-
-    HDFSWriter hdfsWriter = new HDFSWriter() {
-      private volatile boolean open = false;
-
-      public void configure(Context context) {
-      }
-
-      public void sync() throws IOException {
-        if (!open) {
-          throw new IOException("closed");
-        }
-      }
-
-      public void open(String filePath, CompressionCodec codec, CompressionType cType)
-          throws IOException {
-        open = true;
-      }
-
-      public void open(String filePath) throws IOException {
-        open = true;
-      }
-
-      public void close() throws IOException {
-        open = false;
-      }
-
-      @Override
-      public boolean isUnderReplicated() {
-        return false;
-      }
-
-      public void append(Event e) throws IOException {
-        // we just re-open in append if closed
-        open = true;
-      }
-    };
-
-
-    File tmpFile = File.createTempFile("flume", "test");
-    tmpFile.deleteOnExit();
-    String path = tmpFile.getParent();
-    String name = tmpFile.getName();
-
-    BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
-        .setRollInterval(ROLL_INTERVAL)
-        .setFilePath(path)
-        .setFileName(name)
-        .build();
-
-    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
-    for (int i = 0; i < NUM_EVENTS - 1; i++) {
-      bucketWriter.append(e);
-    }
-
-    // sleep to force a roll... wait 2x interval just to be sure
-    Thread.sleep(2 * ROLL_INTERVAL * 1000L);
-
-    bucketWriter.flush(); // throws closed exception
-  }
-
-  @Test
-  public void testFileSuffixNotGiven() throws IOException, InterruptedException {
-    final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
-    final String suffix = null;
-
-    // Need to override system time use for test so we know what to expect
-    final long testTime = System.currentTimeMillis();
-    Clock testClock = new Clock() {
-      public long currentTimeMillis() {
-        return testTime;
-      }
-    };
-
-    MockHDFSWriter hdfsWriter = new MockHDFSWriter();
-    BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
-        .setRollInterval(ROLL_INTERVAL)
-        .setFileSuffix(suffix)
-        .setClock(testClock)
-        .build();
-
-    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
-    bucketWriter.append(e);
-
-    Assert.assertTrue("Incorrect suffix", hdfsWriter.getOpenedFilePath().endsWith(
-        Long.toString(testTime + 1) + ".tmp"));
-  }
-
-  @Test
-  public void testFileSuffixGiven() throws IOException, InterruptedException {
-    final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
-    final String suffix = ".avro";
-
-    // Need to override system time use for test so we know what to expect
-
-    final long testTime = System.currentTimeMillis();
-
-    Clock testClock = new Clock() {
-      public long currentTimeMillis() {
-        return testTime;
-      }
-    };
-
-    MockHDFSWriter hdfsWriter = new MockHDFSWriter();
-    BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
-        .setRollInterval(ROLL_INTERVAL)
-        .setFileSuffix(suffix)
-        .setClock(testClock)
-        .build();
-
-
-    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
-    bucketWriter.append(e);
-
-    Assert.assertTrue("Incorrect suffix",hdfsWriter.getOpenedFilePath().endsWith(
-        Long.toString(testTime + 1) + suffix + ".tmp"));
-  }
-
-  @Test
-  public void testFileSuffixCompressed()
-      throws IOException, InterruptedException {
-    final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
-    final String suffix = ".foo";
-
-    MockHDFSWriter hdfsWriter = new MockHDFSWriter();
-
-    // Need to override system time use for test so we know what to expect
-    final long testTime = System.currentTimeMillis();
-
-    Clock testClock = new Clock() {
-      public long currentTimeMillis() {
-        return testTime;
-      }
-    };
-
-    BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
-        .setRollInterval(ROLL_INTERVAL)
-        .setFileSuffix(suffix)
-        .setCodeC(HDFSEventSink.getCodec("gzip"))
-        .setCompType(SequenceFile.CompressionType.BLOCK)
-        .setClock(testClock)
-        .build();
-
-
-
-
-    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
-    bucketWriter.append(e);
-
-    Assert.assertTrue("Incorrect suffix", hdfsWriter.getOpenedFilePath().endsWith(
-        Long.toString(testTime + 1) + suffix + ".tmp"));
-  }
-
-  @Test
-  public void testInUsePrefix() throws IOException, InterruptedException {
-    final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
-    final String PREFIX = "BRNO_IS_CITY_IN_CZECH_REPUBLIC";
-
-    MockHDFSWriter hdfsWriter = new MockHDFSWriter();
-    HDFSTextSerializer formatter = new HDFSTextSerializer();
-    BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
-        .setRollInterval(ROLL_INTERVAL)
-        .setInUsePrefix(PREFIX)
-        .build();
-
-    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
-    bucketWriter.append(e);
-
-    Assert.assertTrue("Incorrect in use prefix", hdfsWriter.getOpenedFilePath().contains(PREFIX));
-  }
-
-  @Test
-  public void testInUseSuffix() throws IOException, InterruptedException {
-    final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
-    final String SUFFIX = "WELCOME_TO_THE_HELLMOUNTH";
-
-    MockHDFSWriter hdfsWriter = new MockHDFSWriter();
-    HDFSTextSerializer serializer = new HDFSTextSerializer();
-    BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
-        .setRollInterval(ROLL_INTERVAL)
-        .setInUseSuffix(SUFFIX)
-        .build();
-
-    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
-    bucketWriter.append(e);
-
-    Assert.assertTrue("Incorrect in use suffix", hdfsWriter.getOpenedFilePath().contains(SUFFIX));
-  }
-
-  @Test
-  public void testCallbackOnClose() throws IOException, InterruptedException {
-    final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
-    final String SUFFIX = "WELCOME_TO_THE_EREBOR";
-    final AtomicBoolean callbackCalled = new AtomicBoolean(false);
-
-    BucketWriter bucketWriter = new BucketWriterBuilder()
-        .setRollInterval(ROLL_INTERVAL)
-        .setInUseSuffix(SUFFIX)
-        .setOnCloseCallback(new HDFSEventSink.WriterCallback() {
-          @Override
-          public void run(String filePath) {
-            callbackCalled.set(true);
-          }
-        })
-        .setOnCloseCallbackPath("blah")
-        .build();
-
-    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
-    bucketWriter.append(e);
-    bucketWriter.close(true);
-
-    Assert.assertTrue(callbackCalled.get());
-  }
-
-  @Test
-  public void testSequenceFileRenameRetries() throws Exception {
-    sequenceFileRenameRetryCoreTest(1, true);
-    sequenceFileRenameRetryCoreTest(5, true);
-    sequenceFileRenameRetryCoreTest(2, true);
-
-    sequenceFileRenameRetryCoreTest(1, false);
-    sequenceFileRenameRetryCoreTest(5, false);
-    sequenceFileRenameRetryCoreTest(2, false);
-  }
-
-  @Test
-  public void testSequenceFileCloseRetries() throws Exception {
-    sequenceFileCloseRetryCoreTest(5);
-    sequenceFileCloseRetryCoreTest(1);
-  }
-
-  public void sequenceFileRenameRetryCoreTest(int numberOfRetriesRequired, boolean closeSucceed)
-      throws Exception {
-    String hdfsPath = "file:///tmp/flume-test." +
-                      Calendar.getInstance().getTimeInMillis() +
-                      "." + Thread.currentThread().getId();
-
-    Context context = new Context();
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(hdfsPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-    context.put("hdfs.path", hdfsPath);
-    context.put("hdfs.closeTries", String.valueOf(numberOfRetriesRequired));
-    context.put("hdfs.rollCount", "1");
-    context.put("hdfs.retryInterval", "1");
-    context.put("hdfs.callTimeout", Long.toString(1000));
-    MockFileSystem mockFs = new MockFileSystem(fs, numberOfRetriesRequired, closeSucceed);
-    MockDataStream writer = new MockDataStream(mockFs);
-    BucketWriter bucketWriter = new BucketWriterBuilder(writer)
-        .setRollCount(1)
-        .setBatchSize(1)
-        .setFilePath(hdfsPath)
-        .setFileName(hdfsPath)
-        .setInUsePrefix("singleBucket")
-        .setCompType(null)
-        .setRetryInterval(1)
-        .setMaxCloseTries(numberOfRetriesRequired)
-        .setWriter(writer)
-        .build();
-
-    bucketWriter.setFileSystem(mockFs);
-    // At this point, we checked if isFileClosed is available in
-    // this JVM, so lets make it check again.
-    Event event = EventBuilder.withBody("test", Charsets.UTF_8);
-    bucketWriter.append(event);
-    // This is what triggers the close, so a 2nd append is required :/
-    bucketWriter.append(event);
-
-    TimeUnit.SECONDS.sleep(numberOfRetriesRequired + 2);
-
-    Assert.assertTrue("Expected " + numberOfRetriesRequired + " " +
-                      "but got " + bucketWriter.renameTries.get(),
-                      bucketWriter.renameTries.get() == numberOfRetriesRequired);
-  }
-
-  private void sequenceFileCloseRetryCoreTest(int numberOfRetriesRequired)
-      throws Exception {
-    String hdfsPath = "file:///tmp/flume-test." +
-        Calendar.getInstance().getTimeInMillis() +
-        "." + Thread.currentThread().getId();
-
-    Context context = new Context();
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(hdfsPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-    context.put("hdfs.path", hdfsPath);
-    context.put("hdfs.closeTries", String.valueOf(numberOfRetriesRequired));
-    context.put("hdfs.rollCount", "1");
-    context.put("hdfs.retryInterval", "1");
-    context.put("hdfs.callTimeout", Long.toString(1000));
-    MockHDFSWriter mockHDFSWriter = new MockHDFSWriter(Integer.MAX_VALUE);
-    ExecutorService executorService = Executors.newSingleThreadExecutor();
-    BucketWriter bucketWriter = new BucketWriter(
-        0, 0, 1, 1, ctx, hdfsPath, hdfsPath, "singleBucket", ".tmp", null, null,
-        null, mockHDFSWriter, timedRollerPool, proxy,
-        new SinkCounter("test-bucket-writer-" + System.currentTimeMillis()), 0, null, null, 30000,
-        executorService, 1, numberOfRetriesRequired);
-
-    Event event = EventBuilder.withBody("test", Charsets.UTF_8);
-    bucketWriter.append(event);
-    bucketWriter.close(false);
-    TimeUnit.SECONDS.sleep(numberOfRetriesRequired + 2);
-    Assert.assertEquals("ExcceutorService should be empty",
-        executorService.shutdownNow().size(), 0);
-    Assert.assertEquals("Expected " + numberOfRetriesRequired + " " +
-        "but got " + mockHDFSWriter.currentCloseAttempts,
-        mockHDFSWriter.currentCloseAttempts.get(), numberOfRetriesRequired);
-  }
-
-  // Test that we don't swallow IOExceptions in secure mode. We should close the bucket writer
-  // and rethrow the exception. Regression test for FLUME-3049.
-  @Test
-  public void testRotateBucketOnIOException() throws IOException, InterruptedException {
-    MockHDFSWriter hdfsWriter = Mockito.spy(new MockHDFSWriter());
-    PrivilegedExecutor ugiProxy =
-        FlumeAuthenticationUtil.getAuthenticator(null, null).proxyAs("alice");
-
-    final int ROLL_COUNT = 1; // Cause a roll after every successful append().
-    BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
-        .setProxyUser(ugiProxy)
-        .setRollCount(ROLL_COUNT)
-        .build();
-
-    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
-
-    // Write one event successfully.
-    bucketWriter.append(e);
-
-    // Fail the next write.
-    IOException expectedIOException = new IOException("Test injected IOException");
-    Mockito.doThrow(expectedIOException).when(hdfsWriter)
-        .append(Mockito.any(Event.class));
-
-    // The second time we try to write we should get an IOException.
-    try {
-      bucketWriter.append(e);
-      Assert.fail("Expected IOException wasn't thrown during append");
-    } catch (IOException ex) {
-      Assert.assertEquals(expectedIOException, ex);
-      logger.info("Caught expected IOException", ex);
-    }
-
-    // The third time we try to write we should get a BucketClosedException, because the
-    // BucketWriter should attempt to close itself before rethrowing the IOException on the first
-    // call.
-    try {
-      bucketWriter.append(e);
-      Assert.fail("BucketWriter should be already closed, BucketClosedException expected");
-    } catch (BucketClosedException ex) {
-      logger.info("Caught expected BucketClosedException", ex);
-    }
-
-    Assert.assertEquals("events written", 1, hdfsWriter.getEventsWritten());
-    Assert.assertEquals("2 files should be closed", 2, hdfsWriter.getFilesClosed());
-  }
-
-  private class BucketWriterBuilder {
-    private long rollInterval = 0;
-    private long rollSize = 0;
-    private long rollCount = 0;
-    private long batchSize = 0;
-    private Context context = TestBucketWriter.this.ctx;
-    private String filePath = "/tmp";
-    private String fileName = "file";
-    private String inUsePrefix = "";
-    private String inUseSuffix = ".tmp";
-    private String fileSuffix = null;
-    private CompressionCodec codeC = null;
-    private CompressionType compType = SequenceFile.CompressionType.NONE;
-    private HDFSWriter writer = null;
-    private ScheduledExecutorService timedRollerPool = TestBucketWriter.timedRollerPool;
-    private PrivilegedExecutor proxyUser = TestBucketWriter.proxy;
-    private SinkCounter sinkCounter = new SinkCounter(
-        "test-bucket-writer-" + System.currentTimeMillis());
-    private int idleTimeout = 0;
-    private WriterCallback onCloseCallback = null;
-    private String onCloseCallbackPath = null;
-    private long callTimeout = 30000;
-    private ExecutorService callTimeoutPool = Executors.newSingleThreadExecutor();
-    private long retryInterval = 0;
-    private int maxCloseTries = 0;
-    private Clock clock = null;
-
-    public BucketWriterBuilder() {
-    }
-
-    public BucketWriterBuilder(HDFSWriter writer) {
-      this.writer = writer;
-    }
-
-    public BucketWriterBuilder setRollInterval(long rollInterval) {
-      this.rollInterval = rollInterval;
-      return this;
-    }
-
-    public BucketWriterBuilder setRollSize(long rollSize) {
-      this.rollSize = rollSize;
-      return this;
-    }
-
-    public BucketWriterBuilder setRollCount(long rollCount) {
-      this.rollCount = rollCount;
-      return this;
-    }
-
-    public BucketWriterBuilder setBatchSize(long batchSize) {
-      this.batchSize = batchSize;
-      return this;
-    }
-
-    @SuppressWarnings("unused")
-    public BucketWriterBuilder setContext(Context context) {
-      this.context = context;
-      return this;
-    }
-
-    public BucketWriterBuilder setFilePath(String filePath) {
-      this.filePath = filePath;
-      return this;
-    }
-
-    public BucketWriterBuilder setFileName(String fileName) {
-      this.fileName = fileName;
-      return this;
-    }
-
-    public BucketWriterBuilder setInUsePrefix(String inUsePrefix) {
-      this.inUsePrefix = inUsePrefix;
-      return this;
-    }
-
-    public BucketWriterBuilder setInUseSuffix(String inUseSuffix) {
-      this.inUseSuffix = inUseSuffix;
-      return this;
-    }
-
-    public BucketWriterBuilder setFileSuffix(String fileSuffix) {
-      this.fileSuffix = fileSuffix;
-      return this;
-    }
-
-    public BucketWriterBuilder setCodeC(CompressionCodec codeC) {
-      this.codeC = codeC;
-      return this;
-    }
-
-    public BucketWriterBuilder setCompType(CompressionType compType) {
-      this.compType = compType;
-      return this;
-    }
-
-    @SuppressWarnings("unused")
-    public BucketWriterBuilder setTimedRollerPool(
-        ScheduledExecutorService timedRollerPool) {
-      this.timedRollerPool = timedRollerPool;
-      return this;
-    }
-
-    @SuppressWarnings("unused")
-    public BucketWriterBuilder setProxyUser(PrivilegedExecutor proxyUser) {
-      this.proxyUser = proxyUser;
-      return this;
-    }
-
-    @SuppressWarnings("unused")
-    public BucketWriterBuilder setSinkCounter(SinkCounter sinkCounter) {
-      this.sinkCounter = sinkCounter;
-      return this;
-    }
-
-    @SuppressWarnings("unused")
-    public BucketWriterBuilder setIdleTimeout(int idleTimeout) {
-      this.idleTimeout = idleTimeout;
-      return this;
-    }
-
-    public BucketWriterBuilder setOnCloseCallback(
-        WriterCallback onCloseCallback) {
-      this.onCloseCallback = onCloseCallback;
-      return this;
-    }
-
-    public BucketWriterBuilder setOnCloseCallbackPath(
-        String onCloseCallbackPath) {
-      this.onCloseCallbackPath = onCloseCallbackPath;
-      return this;
-    }
-
-    @SuppressWarnings("unused")
-    public BucketWriterBuilder setCallTimeout(long callTimeout) {
-      this.callTimeout = callTimeout;
-      return this;
-    }
-
-    @SuppressWarnings("unused")
-    public BucketWriterBuilder setCallTimeoutPool(
-        ExecutorService callTimeoutPool) {
-      this.callTimeoutPool = callTimeoutPool;
-      return this;
-    }
-
-    public BucketWriterBuilder setRetryInterval(long retryInterval) {
-      this.retryInterval = retryInterval;
-      return this;
-    }
-
-    public BucketWriterBuilder setMaxCloseTries(int maxCloseTries) {
-      this.maxCloseTries = maxCloseTries;
-      return this;
-    }
-
-    public BucketWriterBuilder setWriter(HDFSWriter writer) {
-      this.writer = writer;
-      return this;
-    }
-
-    public BucketWriterBuilder setClock(Clock clock) {
-      this.clock = clock;
-      return this;
-    }
-
-    public BucketWriter build() {
-      if (clock ==  null) {
-        clock = new SystemClock();
-      }
-      if (writer == null) {
-        writer = new MockHDFSWriter();
-      }
-
-      return new BucketWriter(rollInterval, rollSize, rollCount, batchSize,
-          context, filePath, fileName, inUsePrefix, inUseSuffix, fileSuffix,
-          codeC, compType, writer, timedRollerPool, proxyUser, sinkCounter,
-          idleTimeout, onCloseCallback, onCloseCallbackPath, callTimeout,
-          callTimeoutPool, retryInterval, maxCloseTries, clock);
-    }
-  }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSCompressedDataStream.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSCompressedDataStream.java
deleted file mode 100644
index 80f199b7d..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSCompressedDataStream.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.nio.ByteBuffer;
-import java.nio.charset.CharsetDecoder;
-import java.util.List;
-import java.util.zip.GZIPInputStream;
-
-import org.apache.avro.file.DataFileStream;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.io.DatumReader;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.event.EventBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.Lists;
-
-public class TestHDFSCompressedDataStream {
-
-  private static final Logger logger =
-      LoggerFactory.getLogger(TestHDFSCompressedDataStream.class);
-
-  private File file;
-  private String fileURI;
-  private CompressionCodecFactory factory;
-
-  @Before
-  public void init() throws Exception {
-    this.file = new File("target/test/data/foo.gz");
-    this.fileURI = file.getAbsoluteFile().toURI().toString();
-    logger.info("File URI: {}", fileURI);
-
-    Configuration conf = new Configuration();
-    // local FS must be raw in order to be Syncable
-    conf.set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem");
-    Path path = new Path(fileURI);
-    path.getFileSystem(conf); // get FS with our conf cached
-
-    this.factory = new CompressionCodecFactory(conf);
-  }
-
-  // make sure the data makes it to disk if we sync() the data stream
-  @Test
-  public void testGzipDurability() throws Exception {
-    Context context = new Context();
-    HDFSCompressedDataStream writer = new HDFSCompressedDataStream();
-    writer.configure(context);
-    writer.open(fileURI, factory.getCodec(new Path(fileURI)),
-        SequenceFile.CompressionType.BLOCK);
-
-    String[] bodies = { "yarf!" };
-    writeBodies(writer, bodies);
-
-    byte[] buf = new byte[256];
-    GZIPInputStream cmpIn = new GZIPInputStream(new FileInputStream(file));
-    int len = cmpIn.read(buf);
-    String result = new String(buf, 0, len, Charsets.UTF_8);
-    result = result.trim(); // BodyTextEventSerializer adds a newline
-
-    Assert.assertEquals("input and output must match", bodies[0], result);
-  }
-
-  @Test
-  public void testGzipDurabilityWithSerializer() throws Exception {
-    Context context = new Context();
-    context.put("serializer", "AVRO_EVENT");
-
-    HDFSCompressedDataStream writer = new HDFSCompressedDataStream();
-    writer.configure(context);
-
-    writer.open(fileURI, factory.getCodec(new Path(fileURI)),
-        SequenceFile.CompressionType.BLOCK);
-
-    String[] bodies = { "yarf!", "yarfing!" };
-    writeBodies(writer, bodies);
-
-    int found = 0;
-    int expected = bodies.length;
-    List<String> expectedBodies = Lists.newArrayList(bodies);
-
-    GZIPInputStream cmpIn = new GZIPInputStream(new FileInputStream(file));
-    DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
-    DataFileStream<GenericRecord> avroStream =
-        new DataFileStream<GenericRecord>(cmpIn, reader);
-    GenericRecord record = new GenericData.Record(avroStream.getSchema());
-    while (avroStream.hasNext()) {
-      avroStream.next(record);
-      CharsetDecoder decoder = Charsets.UTF_8.newDecoder();
-      String bodyStr = decoder.decode((ByteBuffer) record.get("body"))
-          .toString();
-      expectedBodies.remove(bodyStr);
-      found++;
-    }
-    avroStream.close();
-    cmpIn.close();
-
-    Assert.assertTrue("Found = " + found + ", Expected = " + expected
-        + ", Left = " + expectedBodies.size() + " " + expectedBodies,
-        expectedBodies.size() == 0);
-  }
-
-  private void writeBodies(HDFSCompressedDataStream writer, String... bodies)
-      throws Exception {
-    for (String body : bodies) {
-      Event evt = EventBuilder.withBody(body, Charsets.UTF_8);
-      writer.append(evt);
-    }
-    writer.sync();
-  }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSink.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSink.java
deleted file mode 100644
index 25bd2dab6..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSink.java
+++ /dev/null
@@ -1,1687 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.ByteBuffer;
-import java.nio.charset.CharsetDecoder;
-import java.util.Arrays;
-import java.util.Calendar;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Maps;
-import org.apache.avro.file.DataFileStream;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.io.DatumReader;
-import org.apache.commons.lang.StringUtils;
-import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
-import org.apache.flume.Clock;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.Sink.Status;
-import org.apache.flume.SystemClock;
-import org.apache.flume.Transaction;
-import org.apache.flume.channel.BasicTransactionSemantics;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.conf.Configurables;
-import org.apache.flume.event.EventBuilder;
-import org.apache.flume.event.SimpleEvent;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.flume.lifecycle.LifecycleException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.mockito.internal.util.reflection.Whitebox;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.Lists;
-
-public class TestHDFSEventSink {
-
-  private HDFSEventSink sink;
-  private String testPath;
-  private static final Logger LOG = LoggerFactory
-      .getLogger(HDFSEventSink.class);
-
-  static {
-    System.setProperty("java.security.krb5.realm", "flume");
-    System.setProperty("java.security.krb5.kdc", "blah");
-  }
-
-  private void dirCleanup() {
-    Configuration conf = new Configuration();
-    try {
-      FileSystem fs = FileSystem.get(conf);
-      Path dirPath = new Path(testPath);
-      if (fs.exists(dirPath)) {
-        fs.delete(dirPath, true);
-      }
-    } catch (IOException eIO) {
-      LOG.warn("IO Error in test cleanup", eIO);
-    }
-  }
-
-  // TODO: use System.getProperty("file.separator") instead of hardcoded '/'
-  @Before
-  public void setUp() {
-    LOG.debug("Starting...");
-    /*
-     * FIXME: Use a dynamic path to support concurrent test execution. Also,
-     * beware of the case where this path is used for something or when the
-     * Hadoop config points at file:/// rather than hdfs://. We need to find a
-     * better way of testing HDFS related functionality.
-     */
-    testPath = "file:///tmp/flume-test."
-        + Calendar.getInstance().getTimeInMillis() + "."
-        + Thread.currentThread().getId();
-
-    sink = new HDFSEventSink();
-    sink.setName("HDFSEventSink-" + UUID.randomUUID().toString());
-    dirCleanup();
-  }
-
-  @After
-  public void tearDown() {
-    if (System.getenv("hdfs_keepFiles") == null) dirCleanup();
-  }
-
-  @Test
-  public void testTextBatchAppend() throws Exception {
-    doTestTextBatchAppend(false);
-  }
-
-  @Test
-  public void testTextBatchAppendRawFS() throws Exception {
-    doTestTextBatchAppend(true);
-  }
-
-  public void doTestTextBatchAppend(boolean useRawLocalFileSystem)
-      throws Exception {
-    LOG.debug("Starting...");
-
-    final long rollCount = 10;
-    final long batchSize = 2;
-    final String fileName = "FlumeData";
-    String newPath = testPath + "/singleTextBucket";
-    int totalEvents = 0;
-    int i = 1, j = 1;
-
-    // clear the test directory
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(newPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-
-    Context context = new Context();
-
-    // context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");
-    context.put("hdfs.path", newPath);
-    context.put("hdfs.filePrefix", fileName);
-    context.put("hdfs.rollCount", String.valueOf(rollCount));
-    context.put("hdfs.rollInterval", "0");
-    context.put("hdfs.rollSize", "0");
-    context.put("hdfs.batchSize", String.valueOf(batchSize));
-    context.put("hdfs.writeFormat", "Text");
-    context.put("hdfs.useRawLocalFileSystem",
-        Boolean.toString(useRawLocalFileSystem));
-    context.put("hdfs.fileType", "DataStream");
-
-    Configurables.configure(sink, context);
-
-    Channel channel = new MemoryChannel();
-    Configurables.configure(channel, context);
-
-    sink.setChannel(channel);
-    sink.start();
-
-    Calendar eventDate = Calendar.getInstance();
-    List<String> bodies = Lists.newArrayList();
-
-    // push the event batches into channel to roll twice
-    for (i = 1; i <= (rollCount * 10) / batchSize; i++) {
-      Transaction txn = channel.getTransaction();
-      txn.begin();
-      for (j = 1; j <= batchSize; j++) {
-        Event event = new SimpleEvent();
-        eventDate.clear();
-        eventDate.set(2011, i, i, i, 0); // yy mm dd
-        String body = "Test." + i + "." + j;
-        event.setBody(body.getBytes());
-        bodies.add(body);
-        channel.put(event);
-        totalEvents++;
-      }
-      txn.commit();
-      txn.close();
-
-      // execute sink to process the events
-      sink.process();
-    }
-
-    sink.stop();
-
-    // loop through all the files generated and check their contains
-    FileStatus[] dirStat = fs.listStatus(dirPath);
-    Path[] fList = FileUtil.stat2Paths(dirStat);
-
-    // check that the roll happened correctly for the given data
-    long expectedFiles = totalEvents / rollCount;
-    if (totalEvents % rollCount > 0) expectedFiles++;
-    Assert.assertEquals("num files wrong, found: " +
-        Lists.newArrayList(fList), expectedFiles, fList.length);
-    // check the contents of the all files
-    verifyOutputTextFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
-  }
-
-  @Test
-  public void testLifecycle() throws InterruptedException, LifecycleException {
-    LOG.debug("Starting...");
-    Context context = new Context();
-
-    context.put("hdfs.path", testPath);
-    /*
-     * context.put("hdfs.rollInterval", String.class);
-     * context.get("hdfs.rollSize", String.class); context.get("hdfs.rollCount",
-     * String.class);
-     */
-    Configurables.configure(sink, context);
-
-    sink.setChannel(new MemoryChannel());
-
-    sink.start();
-    sink.stop();
-  }
-
-  @Test
-  public void testEmptyChannelResultsInStatusBackoff()
-      throws InterruptedException, LifecycleException, EventDeliveryException {
-    LOG.debug("Starting...");
-    Context context = new Context();
-    Channel channel = new MemoryChannel();
-    context.put("hdfs.path", testPath);
-    context.put("keep-alive", "0");
-    Configurables.configure(sink, context);
-    Configurables.configure(channel, context);
-    sink.setChannel(channel);
-    sink.start();
-    Assert.assertEquals(Status.BACKOFF, sink.process());
-    sink.stop();
-  }
-
-  @Test
-  public void testKerbFileAccess() throws InterruptedException,
-      LifecycleException, EventDeliveryException, IOException {
-    LOG.debug("Starting testKerbFileAccess() ...");
-    final String fileName = "FlumeData";
-    final long rollCount = 5;
-    final long batchSize = 2;
-    String newPath = testPath + "/singleBucket";
-    String kerbConfPrincipal = "user1/localhost@EXAMPLE.COM";
-    String kerbKeytab = "/usr/lib/flume/nonexistkeytabfile";
-
-    //turn security on
-    Configuration conf = new Configuration();
-    conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
-        "kerberos");
-    UserGroupInformation.setConfiguration(conf);
-
-    Context context = new Context();
-    context.put("hdfs.path", newPath);
-    context.put("hdfs.filePrefix", fileName);
-    context.put("hdfs.rollCount", String.valueOf(rollCount));
-    context.put("hdfs.batchSize", String.valueOf(batchSize));
-    context.put("hdfs.kerberosPrincipal", kerbConfPrincipal);
-    context.put("hdfs.kerberosKeytab", kerbKeytab);
-
-    try {
-      Configurables.configure(sink, context);
-      Assert.fail("no exception thrown");
-    } catch (IllegalArgumentException expected) {
-      Assert.assertTrue(expected.getMessage().contains(
-          "Keytab is not a readable file"));
-    } finally {
-      //turn security off
-      conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
-          "simple");
-      UserGroupInformation.setConfiguration(conf);
-    }
-  }
-
-  @Test
-  public void testTextAppend() throws InterruptedException, LifecycleException,
-      EventDeliveryException, IOException {
-
-    LOG.debug("Starting...");
-    final long rollCount = 3;
-    final long batchSize = 2;
-    final String fileName = "FlumeData";
-    String newPath = testPath + "/singleTextBucket";
-    int totalEvents = 0;
-    int i = 1, j = 1;
-
-    // clear the test directory
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(newPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-
-    Context context = new Context();
-
-    // context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");
-    context.put("hdfs.path", newPath);
-    context.put("hdfs.filePrefix", fileName);
-    context.put("hdfs.rollCount", String.valueOf(rollCount));
-    context.put("hdfs.batchSize", String.valueOf(batchSize));
-    context.put("hdfs.writeFormat", "Text");
-    context.put("hdfs.fileType", "DataStream");
-
-    Configurables.configure(sink, context);
-
-    Channel channel = new MemoryChannel();
-    Configurables.configure(channel, context);
-
-    sink.setChannel(channel);
-    sink.start();
-
-    Calendar eventDate = Calendar.getInstance();
-    List<String> bodies = Lists.newArrayList();
-
-    // push the event batches into channel
-    for (i = 1; i < 4; i++) {
-      Transaction txn = channel.getTransaction();
-      txn.begin();
-      for (j = 1; j <= batchSize; j++) {
-        Event event = new SimpleEvent();
-        eventDate.clear();
-        eventDate.set(2011, i, i, i, 0); // yy mm dd
-        event.getHeaders().put("timestamp",
-            String.valueOf(eventDate.getTimeInMillis()));
-        event.getHeaders().put("hostname", "Host" + i);
-        String body = "Test." + i + "." + j;
-        event.setBody(body.getBytes());
-        bodies.add(body);
-        channel.put(event);
-        totalEvents++;
-      }
-      txn.commit();
-      txn.close();
-
-      // execute sink to process the events
-      sink.process();
-    }
-
-    sink.stop();
-
-    // loop through all the files generated and check their contains
-    FileStatus[] dirStat = fs.listStatus(dirPath);
-    Path[] fList = FileUtil.stat2Paths(dirStat);
-
-    // check that the roll happened correctly for the given data
-    long expectedFiles = totalEvents / rollCount;
-    if (totalEvents % rollCount > 0) expectedFiles++;
-    Assert.assertEquals("num files wrong, found: " +
-        Lists.newArrayList(fList), expectedFiles, fList.length);
-    verifyOutputTextFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
-  }
-
-  @Test
-  public void testAvroAppend() throws InterruptedException, LifecycleException,
-      EventDeliveryException, IOException {
-
-    LOG.debug("Starting...");
-    final long rollCount = 3;
-    final long batchSize = 2;
-    final String fileName = "FlumeData";
-    String newPath = testPath + "/singleTextBucket";
-    int totalEvents = 0;
-    int i = 1, j = 1;
-
-    // clear the test directory
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(newPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-
-    Context context = new Context();
-
-    // context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");
-    context.put("hdfs.path", newPath);
-    context.put("hdfs.filePrefix", fileName);
-    context.put("hdfs.rollCount", String.valueOf(rollCount));
-    context.put("hdfs.batchSize", String.valueOf(batchSize));
-    context.put("hdfs.writeFormat", "Text");
-    context.put("hdfs.fileType", "DataStream");
-    context.put("serializer", "AVRO_EVENT");
-
-    Configurables.configure(sink, context);
-
-    Channel channel = new MemoryChannel();
-    Configurables.configure(channel, context);
-
-    sink.setChannel(channel);
-    sink.start();
-
-    Calendar eventDate = Calendar.getInstance();
-    List<String> bodies = Lists.newArrayList();
-
-    // push the event batches into channel
-    for (i = 1; i < 4; i++) {
-      Transaction txn = channel.getTransaction();
-      txn.begin();
-      for (j = 1; j <= batchSize; j++) {
-        Event event = new SimpleEvent();
-        eventDate.clear();
-        eventDate.set(2011, i, i, i, 0); // yy mm dd
-        event.getHeaders().put("timestamp",
-            String.valueOf(eventDate.getTimeInMillis()));
-        event.getHeaders().put("hostname", "Host" + i);
-        String body = "Test." + i + "." + j;
-        event.setBody(body.getBytes());
-        bodies.add(body);
-        channel.put(event);
-        totalEvents++;
-      }
-      txn.commit();
-      txn.close();
-
-      // execute sink to process the events
-      sink.process();
-    }
-
-    sink.stop();
-
-    // loop through all the files generated and check their contains
-    FileStatus[] dirStat = fs.listStatus(dirPath);
-    Path[] fList = FileUtil.stat2Paths(dirStat);
-
-    // check that the roll happened correctly for the given data
-    long expectedFiles = totalEvents / rollCount;
-    if (totalEvents % rollCount > 0) expectedFiles++;
-    Assert.assertEquals("num files wrong, found: " +
-        Lists.newArrayList(fList), expectedFiles, fList.length);
-    verifyOutputAvroFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
-  }
-
-  @Test
-  public void testSimpleAppend() throws InterruptedException,
-      LifecycleException, EventDeliveryException, IOException {
-
-    LOG.debug("Starting...");
-    final String fileName = "FlumeData";
-    final long rollCount = 5;
-    final long batchSize = 2;
-    final int numBatches = 4;
-    String newPath = testPath + "/singleBucket";
-    int totalEvents = 0;
-    int i = 1, j = 1;
-
-    // clear the test directory
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(newPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-
-    Context context = new Context();
-
-    context.put("hdfs.path", newPath);
-    context.put("hdfs.filePrefix", fileName);
-    context.put("hdfs.rollCount", String.valueOf(rollCount));
-    context.put("hdfs.batchSize", String.valueOf(batchSize));
-
-    Configurables.configure(sink, context);
-
-    Channel channel = new MemoryChannel();
-    Configurables.configure(channel, context);
-
-    sink.setChannel(channel);
-    sink.start();
-
-    Calendar eventDate = Calendar.getInstance();
-    List<String> bodies = Lists.newArrayList();
-
-    // push the event batches into channel
-    for (i = 1; i < numBatches; i++) {
-      Transaction txn = channel.getTransaction();
-      txn.begin();
-      for (j = 1; j <= batchSize; j++) {
-        Event event = new SimpleEvent();
-        eventDate.clear();
-        eventDate.set(2011, i, i, i, 0); // yy mm dd
-        event.getHeaders().put("timestamp",
-            String.valueOf(eventDate.getTimeInMillis()));
-        event.getHeaders().put("hostname", "Host" + i);
-        String body = "Test." + i + "." + j;
-        event.setBody(body.getBytes());
-        bodies.add(body);
-        channel.put(event);
-        totalEvents++;
-      }
-      txn.commit();
-      txn.close();
-
-      // execute sink to process the events
-      sink.process();
-    }
-
-    sink.stop();
-
-    // loop through all the files generated and check their contains
-    FileStatus[] dirStat = fs.listStatus(dirPath);
-    Path[] fList = FileUtil.stat2Paths(dirStat);
-
-    // check that the roll happened correctly for the given data
-    long expectedFiles = totalEvents / rollCount;
-    if (totalEvents % rollCount > 0) expectedFiles++;
-    Assert.assertEquals("num files wrong, found: " +
-        Lists.newArrayList(fList), expectedFiles, fList.length);
-    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
-  }
-
-  @Test
-  public void testSimpleAppendLocalTime()
-      throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
-    final long currentTime = System.currentTimeMillis();
-    Clock clk = new Clock() {
-      @Override
-      public long currentTimeMillis() {
-        return currentTime;
-      }
-    };
-
-    LOG.debug("Starting...");
-    final String fileName = "FlumeData";
-    final long rollCount = 5;
-    final long batchSize = 2;
-    final int numBatches = 4;
-    String newPath = testPath + "/singleBucket/%s" ;
-    String expectedPath = testPath + "/singleBucket/" +
-        String.valueOf(currentTime / 1000);
-    int totalEvents = 0;
-    int i = 1, j = 1;
-
-    // clear the test directory
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(expectedPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-
-    Context context = new Context();
-
-    context.put("hdfs.path", newPath);
-    context.put("hdfs.filePrefix", fileName);
-    context.put("hdfs.rollCount", String.valueOf(rollCount));
-    context.put("hdfs.batchSize", String.valueOf(batchSize));
-    context.put("hdfs.useLocalTimeStamp", String.valueOf(true));
-
-    Configurables.configure(sink, context);
-
-    Channel channel = new MemoryChannel();
-    Configurables.configure(channel, context);
-
-    sink.setChannel(channel);
-    sink.setBucketClock(clk);
-    sink.start();
-
-    Calendar eventDate = Calendar.getInstance();
-    List<String> bodies = Lists.newArrayList();
-
-    // push the event batches into channel
-    for (i = 1; i < numBatches; i++) {
-      Transaction txn = channel.getTransaction();
-      txn.begin();
-      for (j = 1; j <= batchSize; j++) {
-        Event event = new SimpleEvent();
-        eventDate.clear();
-        eventDate.set(2011, i, i, i, 0); // yy mm dd
-        event.getHeaders().put("timestamp",
-            String.valueOf(eventDate.getTimeInMillis()));
-        event.getHeaders().put("hostname", "Host" + i);
-        String body = "Test." + i + "." + j;
-        event.setBody(body.getBytes());
-        bodies.add(body);
-        channel.put(event);
-        totalEvents++;
-      }
-      txn.commit();
-      txn.close();
-
-      // execute sink to process the events
-      sink.process();
-    }
-
-    sink.stop();
-
-    // loop through all the files generated and check their contains
-    FileStatus[] dirStat = fs.listStatus(dirPath);
-    Path[] fList = FileUtil.stat2Paths(dirStat);
-
-    // check that the roll happened correctly for the given data
-    long expectedFiles = totalEvents / rollCount;
-    if (totalEvents % rollCount > 0) expectedFiles++;
-    Assert.assertEquals("num files wrong, found: " +
-        Lists.newArrayList(fList), expectedFiles, fList.length);
-    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
-    // The clock in bucketpath is static, so restore the real clock
-    sink.setBucketClock(new SystemClock());
-  }
-
-  @Test
-  public void testAppend() throws InterruptedException, LifecycleException,
-      EventDeliveryException, IOException {
-
-    LOG.debug("Starting...");
-    final long rollCount = 3;
-    final long batchSize = 2;
-    final String fileName = "FlumeData";
-
-    // clear the test directory
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(testPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-
-    Context context = new Context();
-
-    context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");
-    context.put("hdfs.timeZone", "UTC");
-    context.put("hdfs.filePrefix", fileName);
-    context.put("hdfs.rollCount", String.valueOf(rollCount));
-    context.put("hdfs.batchSize", String.valueOf(batchSize));
-
-    Configurables.configure(sink, context);
-
-    Channel channel = new MemoryChannel();
-    Configurables.configure(channel, context);
-
-    sink.setChannel(channel);
-    sink.start();
-
-    Calendar eventDate = Calendar.getInstance();
-    List<String> bodies = Lists.newArrayList();
-    // push the event batches into channel
-    for (int i = 1; i < 4; i++) {
-      Transaction txn = channel.getTransaction();
-      txn.begin();
-      for (int j = 1; j <= batchSize; j++) {
-        Event event = new SimpleEvent();
-        eventDate.clear();
-        eventDate.set(2011, i, i, i, 0); // yy mm dd
-        event.getHeaders().put("timestamp",
-            String.valueOf(eventDate.getTimeInMillis()));
-        event.getHeaders().put("hostname", "Host" + i);
-        String body = "Test." + i + "." + j;
-        event.setBody(body.getBytes());
-        bodies.add(body);
-        channel.put(event);
-      }
-      txn.commit();
-      txn.close();
-
-      // execute sink to process the events
-      sink.process();
-    }
-
-    sink.stop();
-    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
-  }
-
-  // inject fault and make sure that the txn is rolled back and retried
-  @Test
-  public void testBadSimpleAppend() throws InterruptedException,
-      LifecycleException, EventDeliveryException, IOException {
-
-    LOG.debug("Starting...");
-    final String fileName = "FlumeData";
-    final long rollCount = 5;
-    final long batchSize = 2;
-    final int numBatches = 4;
-    String newPath = testPath + "/singleBucket";
-    int totalEvents = 0;
-    int i = 1, j = 1;
-
-    HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
-    sink = new HDFSEventSink(badWriterFactory);
-
-    // clear the test directory
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(newPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-
-    Context context = new Context();
-
-    context.put("hdfs.path", newPath);
-    context.put("hdfs.filePrefix", fileName);
-    context.put("hdfs.rollCount", String.valueOf(rollCount));
-    context.put("hdfs.batchSize", String.valueOf(batchSize));
-    context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
-
-    Configurables.configure(sink, context);
-
-    Channel channel = new MemoryChannel();
-    Configurables.configure(channel, context);
-
-    sink.setChannel(channel);
-    sink.start();
-
-    Calendar eventDate = Calendar.getInstance();
-
-    List<String> bodies = Lists.newArrayList();
-    // push the event batches into channel
-    for (i = 1; i < numBatches; i++) {
-      Transaction txn = channel.getTransaction();
-      txn.begin();
-      for (j = 1; j <= batchSize; j++) {
-        Event event = new SimpleEvent();
-        eventDate.clear();
-        eventDate.set(2011, i, i, i, 0); // yy mm dd
-        event.getHeaders().put("timestamp",
-            String.valueOf(eventDate.getTimeInMillis()));
-        event.getHeaders().put("hostname", "Host" + i);
-
-        String body = "Test." + i + "." + j;
-        event.setBody(body.getBytes());
-        bodies.add(body);
-        // inject fault
-        if ((totalEvents % 30) == 1) {
-          event.getHeaders().put("fault-once", "");
-        }
-        channel.put(event);
-        totalEvents++;
-      }
-      txn.commit();
-      txn.close();
-
-      LOG.info("Process events: " + sink.process());
-    }
-    LOG.info("Process events to end of transaction max: " + sink.process());
-    LOG.info("Process events to injected fault: " + sink.process());
-    LOG.info("Process events remaining events: " + sink.process());
-    sink.stop();
-    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
-
-    SinkCounter sc = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
-    Assert.assertEquals(1, sc.getEventWriteFail());
-  }
-
-
-  private List<String> getAllFiles(String input) {
-    List<String> output = Lists.newArrayList();
-    File dir = new File(input);
-    if (dir.isFile()) {
-      output.add(dir.getAbsolutePath());
-    } else if (dir.isDirectory()) {
-      for (String file : dir.list()) {
-        File subDir = new File(dir, file);
-        output.addAll(getAllFiles(subDir.getAbsolutePath()));
-      }
-    }
-    return output;
-  }
-
-  private void verifyOutputSequenceFiles(FileSystem fs, Configuration conf, String dir,
-                                         String prefix, List<String> bodies) throws IOException {
-    int found = 0;
-    int expected = bodies.size();
-    for (String outputFile : getAllFiles(dir)) {
-      String name = (new File(outputFile)).getName();
-      if (name.startsWith(prefix)) {
-        SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(outputFile), conf);
-        LongWritable key = new LongWritable();
-        BytesWritable value = new BytesWritable();
-        while (reader.next(key, value)) {
-          String body = new String(value.getBytes(), 0, value.getLength());
-          if (bodies.contains(body)) {
-            LOG.debug("Found event body: {}", body);
-            bodies.remove(body);
-            found++;
-          }
-        }
-        reader.close();
-      }
-    }
-    if (!bodies.isEmpty()) {
-      for (String body : bodies) {
-        LOG.error("Never found event body: {}", body);
-      }
-    }
-    Assert.assertTrue("Found = " + found + ", Expected = "  +
-        expected + ", Left = " + bodies.size() + " " + bodies,
-          bodies.size() == 0);
-
-  }
-
-  private void verifyOutputTextFiles(FileSystem fs, Configuration conf, String dir, String prefix,
-                                     List<String> bodies) throws IOException {
-    int found = 0;
-    int expected = bodies.size();
-    for (String outputFile : getAllFiles(dir)) {
-      String name = (new File(outputFile)).getName();
-      if (name.startsWith(prefix)) {
-        FSDataInputStream input = fs.open(new Path(outputFile));
-        BufferedReader reader = new BufferedReader(new InputStreamReader(input));
-        String body = null;
-        while ((body = reader.readLine()) != null) {
-          bodies.remove(body);
-          found++;
-        }
-        reader.close();
-      }
-    }
-    Assert.assertTrue("Found = " + found + ", Expected = "  +
-        expected + ", Left = " + bodies.size() + " " + bodies,
-          bodies.size() == 0);
-
-  }
-
-  private void verifyOutputAvroFiles(FileSystem fs, Configuration conf, String dir, String prefix,
-                                     List<String> bodies) throws IOException {
-    int found = 0;
-    int expected = bodies.size();
-    for (String outputFile : getAllFiles(dir)) {
-      String name = (new File(outputFile)).getName();
-      if (name.startsWith(prefix)) {
-        FSDataInputStream input = fs.open(new Path(outputFile));
-        DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
-        DataFileStream<GenericRecord> avroStream =
-            new DataFileStream<GenericRecord>(input, reader);
-        GenericRecord record = new GenericData.Record(avroStream.getSchema());
-        while (avroStream.hasNext()) {
-          avroStream.next(record);
-          ByteBuffer body = (ByteBuffer) record.get("body");
-          CharsetDecoder decoder = Charsets.UTF_8.newDecoder();
-          String bodyStr = decoder.decode(body).toString();
-          LOG.debug("Removing event: {}", bodyStr);
-          bodies.remove(bodyStr);
-          found++;
-        }
-        avroStream.close();
-        input.close();
-      }
-    }
-    Assert.assertTrue("Found = " + found + ", Expected = "  +
-        expected + ", Left = " + bodies.size() + " " + bodies,
-            bodies.size() == 0);
-  }
-
-  /**
-   * Ensure that when a write throws an IOException we are
-   * able to continue to progress in the next process() call.
-   * This relies on Transactional rollback semantics for durability and
-   * the behavior of the BucketWriter class of close()ing upon IOException.
-   */
-  @Test
-  public void testCloseReopen()
-      throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
-
-    LOG.debug("Starting...");
-    final int numBatches = 4;
-    final String fileName = "FlumeData";
-    final long rollCount = 5;
-    final long batchSize = 2;
-    String newPath = testPath + "/singleBucket";
-    int i = 1, j = 1;
-
-    HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
-    sink = new HDFSEventSink(badWriterFactory);
-
-    // clear the test directory
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(newPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-
-    Context context = new Context();
-
-    context.put("hdfs.path", newPath);
-    context.put("hdfs.filePrefix", fileName);
-    context.put("hdfs.rollCount", String.valueOf(rollCount));
-    context.put("hdfs.batchSize", String.valueOf(batchSize));
-    context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
-
-    Configurables.configure(sink, context);
-
-    MemoryChannel channel = new MemoryChannel();
-    Configurables.configure(channel, new Context());
-
-    sink.setChannel(channel);
-    sink.start();
-
-    Calendar eventDate = Calendar.getInstance();
-    List<String> bodies = Lists.newArrayList();
-    // push the event batches into channel
-    for (i = 1; i < numBatches; i++) {
-      channel.getTransaction().begin();
-      try {
-        for (j = 1; j <= batchSize; j++) {
-          Event event = new SimpleEvent();
-          eventDate.clear();
-          eventDate.set(2011, i, i, i, 0); // yy mm dd
-          event.getHeaders().put("timestamp",
-              String.valueOf(eventDate.getTimeInMillis()));
-          event.getHeaders().put("hostname", "Host" + i);
-          String body = "Test." + i + "." + j;
-          event.setBody(body.getBytes());
-          bodies.add(body);
-          // inject fault
-          event.getHeaders().put("fault-until-reopen", "");
-          channel.put(event);
-        }
-        channel.getTransaction().commit();
-      } finally {
-        channel.getTransaction().close();
-      }
-      LOG.info("execute sink to process the events: " + sink.process());
-    }
-    LOG.info("clear any events pending due to errors: " + sink.process());
-    sink.stop();
-
-    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
-
-    SinkCounter sc = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
-    Assert.assertEquals(1, sc.getEventWriteFail());
-  }
-
-  /**
-   * Test that the old bucket writer is closed at the end of rollInterval and
-   * a new one is used for the next set of events.
-   */
-  @Test
-  public void testCloseReopenOnRollTime()
-      throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
-
-    LOG.debug("Starting...");
-    final int numBatches = 4;
-    final String fileName = "FlumeData";
-    final long batchSize = 2;
-    String newPath = testPath + "/singleBucket";
-    int i = 1, j = 1;
-
-    HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
-    sink = new HDFSEventSink(badWriterFactory);
-
-    // clear the test directory
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(newPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-
-    Context context = new Context();
-
-    context.put("hdfs.path", newPath);
-    context.put("hdfs.filePrefix", fileName);
-    context.put("hdfs.rollCount", String.valueOf(0));
-    context.put("hdfs.rollSize", String.valueOf(0));
-    context.put("hdfs.rollInterval", String.valueOf(2));
-    context.put("hdfs.batchSize", String.valueOf(batchSize));
-    context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
-
-    Configurables.configure(sink, context);
-
-    MemoryChannel channel = new MemoryChannel();
-    Configurables.configure(channel, new Context());
-
-    sink.setChannel(channel);
-    sink.start();
-
-    Calendar eventDate = Calendar.getInstance();
-    List<String> bodies = Lists.newArrayList();
-    // push the event batches into channel
-    for (i = 1; i < numBatches; i++) {
-      channel.getTransaction().begin();
-      try {
-        for (j = 1; j <= batchSize; j++) {
-          Event event = new SimpleEvent();
-          eventDate.clear();
-          eventDate.set(2011, i, i, i, 0); // yy mm dd
-          event.getHeaders().put("timestamp",
-              String.valueOf(eventDate.getTimeInMillis()));
-          event.getHeaders().put("hostname", "Host" + i);
-          String body = "Test." + i + "." + j;
-          event.setBody(body.getBytes());
-          bodies.add(body);
-          // inject fault
-          event.getHeaders().put("count-check", "");
-          channel.put(event);
-        }
-        channel.getTransaction().commit();
-      } finally {
-        channel.getTransaction().close();
-      }
-      LOG.info("execute sink to process the events: " + sink.process());
-      // Make sure the first file gets rolled due to rollTimeout.
-      if (i == 1) {
-        Thread.sleep(2001);
-      }
-    }
-    LOG.info("clear any events pending due to errors: " + sink.process());
-    sink.stop();
-
-    Assert.assertTrue(badWriterFactory.openCount.get() >= 2);
-    LOG.info("Total number of bucket writers opened: {}",
-        badWriterFactory.openCount.get());
-    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName,
-        bodies);
-  }
-
-  /**
-   * Test that a close due to roll interval removes the bucketwriter from
-   * sfWriters map.
-   */
-  @Test
-  public void testCloseRemovesFromSFWriters()
-      throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
-
-    LOG.debug("Starting...");
-    final String fileName = "FlumeData";
-    final long batchSize = 2;
-    String newPath = testPath + "/singleBucket";
-    int i = 1, j = 1;
-
-    HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
-    sink = new HDFSEventSink(badWriterFactory);
-
-    // clear the test directory
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(newPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-
-    Context context = new Context();
-
-    context.put("hdfs.path", newPath);
-    context.put("hdfs.filePrefix", fileName);
-    context.put("hdfs.rollCount", String.valueOf(0));
-    context.put("hdfs.rollSize", String.valueOf(0));
-    context.put("hdfs.rollInterval", String.valueOf(1));
-    context.put("hdfs.batchSize", String.valueOf(batchSize));
-    context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
-    String expectedLookupPath = newPath + "/FlumeData";
-
-    Configurables.configure(sink, context);
-
-    MemoryChannel channel = new MemoryChannel();
-    Configurables.configure(channel, new Context());
-
-    sink.setChannel(channel);
-    sink.start();
-
-    Calendar eventDate = Calendar.getInstance();
-    List<String> bodies = Lists.newArrayList();
-    // push the event batches into channel
-    channel.getTransaction().begin();
-    try {
-      for (j = 1; j <= 2 * batchSize; j++) {
-        Event event = new SimpleEvent();
-        eventDate.clear();
-        eventDate.set(2011, i, i, i, 0); // yy mm dd
-        event.getHeaders().put("timestamp",
-            String.valueOf(eventDate.getTimeInMillis()));
-        event.getHeaders().put("hostname", "Host" + i);
-        String body = "Test." + i + "." + j;
-        event.setBody(body.getBytes());
-        bodies.add(body);
-        // inject fault
-        event.getHeaders().put("count-check", "");
-        channel.put(event);
-      }
-      channel.getTransaction().commit();
-    } finally {
-      channel.getTransaction().close();
-    }
-    LOG.info("execute sink to process the events: " + sink.process());
-    Assert.assertTrue(sink.getSfWriters().containsKey(expectedLookupPath));
-    // Make sure the first file gets rolled due to rollTimeout.
-    Thread.sleep(2001);
-    Assert.assertFalse(sink.getSfWriters().containsKey(expectedLookupPath));
-    LOG.info("execute sink to process the events: " + sink.process());
-    // A new bucket writer should have been created for this bucket. So
-    // sfWriters map should not have the same key again.
-    Assert.assertTrue(sink.getSfWriters().containsKey(expectedLookupPath));
-    sink.stop();
-
-    LOG.info("Total number of bucket writers opened: {}",
-        badWriterFactory.openCount.get());
-    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName,
-        bodies);
-  }
-
-
-
-  /*
-   * append using slow sink writer.
-   * verify that the process returns backoff due to timeout
-   */
-  @Test
-  public void testSlowAppendFailure() throws InterruptedException,
-      LifecycleException, EventDeliveryException, IOException {
-
-    LOG.debug("Starting...");
-    final String fileName = "FlumeData";
-    final long rollCount = 5;
-    final long batchSize = 2;
-    final int numBatches = 2;
-    String newPath = testPath + "/singleBucket";
-    int i = 1, j = 1;
-
-    // clear the test directory
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(newPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-
-    // create HDFS sink with slow writer
-    HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
-    sink = new HDFSEventSink(badWriterFactory);
-
-    Context context = new Context();
-    context.put("hdfs.path", newPath);
-    context.put("hdfs.filePrefix", fileName);
-    context.put("hdfs.rollCount", String.valueOf(rollCount));
-    context.put("hdfs.batchSize", String.valueOf(batchSize));
-    context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
-    context.put("hdfs.callTimeout", Long.toString(1000));
-    Configurables.configure(sink, context);
-
-    Channel channel = new MemoryChannel();
-    Configurables.configure(channel, context);
-
-    sink.setChannel(channel);
-    sink.start();
-
-    Calendar eventDate = Calendar.getInstance();
-
-    // push the event batches into channel
-    for (i = 0; i < numBatches; i++) {
-      Transaction txn = channel.getTransaction();
-      txn.begin();
-      for (j = 1; j <= batchSize; j++) {
-        Event event = new SimpleEvent();
-        eventDate.clear();
-        eventDate.set(2011, i, i, i, 0); // yy mm dd
-        event.getHeaders().put("timestamp",
-            String.valueOf(eventDate.getTimeInMillis()));
-        event.getHeaders().put("hostname", "Host" + i);
-        event.getHeaders().put("slow", "1500");
-        event.setBody(("Test." + i + "." + j).getBytes());
-        channel.put(event);
-      }
-      txn.commit();
-      txn.close();
-
-      // execute sink to process the events
-      Status satus = sink.process();
-
-      // verify that the append returned backoff due to timeotu
-      Assert.assertEquals(satus, Status.BACKOFF);
-    }
-
-    sink.stop();
-
-    SinkCounter sc = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
-    Assert.assertEquals(2, sc.getEventWriteFail());
-  }
-
-  /*
-   * append using slow sink writer with specified append timeout
-   * verify that the data is written correctly to files
-   */
-  private void slowAppendTestHelper(long appendTimeout)
-      throws InterruptedException, IOException, LifecycleException, EventDeliveryException,
-             IOException {
-    final String fileName = "FlumeData";
-    final long rollCount = 5;
-    final long batchSize = 2;
-    final int numBatches = 2;
-    String newPath = testPath + "/singleBucket";
-    int totalEvents = 0;
-    int i = 1, j = 1;
-
-    // clear the test directory
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(newPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-
-    // create HDFS sink with slow writer
-    HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
-    sink = new HDFSEventSink(badWriterFactory);
-
-    Context context = new Context();
-    context.put("hdfs.path", newPath);
-    context.put("hdfs.filePrefix", fileName);
-    context.put("hdfs.rollCount", String.valueOf(rollCount));
-    context.put("hdfs.batchSize", String.valueOf(batchSize));
-    context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
-    context.put("hdfs.appendTimeout", String.valueOf(appendTimeout));
-    Configurables.configure(sink, context);
-
-    Channel channel = new MemoryChannel();
-    Configurables.configure(channel, context);
-
-    sink.setChannel(channel);
-    sink.start();
-
-    Calendar eventDate = Calendar.getInstance();
-    List<String> bodies = Lists.newArrayList();
-    // push the event batches into channel
-    for (i = 0; i < numBatches; i++) {
-      Transaction txn = channel.getTransaction();
-      txn.begin();
-      for (j = 1; j <= batchSize; j++) {
-        Event event = new SimpleEvent();
-        eventDate.clear();
-        eventDate.set(2011, i, i, i, 0); // yy mm dd
-        event.getHeaders().put("timestamp",
-            String.valueOf(eventDate.getTimeInMillis()));
-        event.getHeaders().put("hostname", "Host" + i);
-        event.getHeaders().put("slow", "1500");
-        String body = "Test." + i + "." + j;
-        event.setBody(body.getBytes());
-        bodies.add(body);
-        channel.put(event);
-        totalEvents++;
-      }
-      txn.commit();
-      txn.close();
-
-      // execute sink to process the events
-      sink.process();
-    }
-
-    sink.stop();
-
-    // loop through all the files generated and check their contains
-    FileStatus[] dirStat = fs.listStatus(dirPath);
-    Path[] fList = FileUtil.stat2Paths(dirStat);
-
-    // check that the roll happened correctly for the given data
-    // Note that we'll end up with two files with only a head
-    long expectedFiles = totalEvents / rollCount;
-    if (totalEvents % rollCount > 0) expectedFiles++;
-    Assert.assertEquals("num files wrong, found: " +
-        Lists.newArrayList(fList), expectedFiles, fList.length);
-    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
-  }
-
-  /*
-   * append using slow sink writer with long append timeout
-   * verify that the data is written correctly to files
-   */
-  @Test
-  public void testSlowAppendWithLongTimeout() throws InterruptedException,
-      LifecycleException, EventDeliveryException, IOException {
-    LOG.debug("Starting...");
-    slowAppendTestHelper(3000);
-  }
-
-  /*
-   * append using slow sink writer with no timeout to make append
-   * synchronous. Verify that the data is written correctly to files
-   */
-  @Test
-  public void testSlowAppendWithoutTimeout() throws InterruptedException,
-      LifecycleException, EventDeliveryException, IOException {
-    LOG.debug("Starting...");
-    slowAppendTestHelper(0);
-  }
-  @Test
-  public void testCloseOnIdle() throws IOException, EventDeliveryException, InterruptedException {
-    String hdfsPath = testPath + "/idleClose";
-
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(hdfsPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-    Context context = new Context();
-    context.put("hdfs.path", hdfsPath);
-    /*
-     * All three rolling methods are disabled so the only
-     * way a file can roll is through the idle timeout.
-     */
-    context.put("hdfs.rollCount", "0");
-    context.put("hdfs.rollSize", "0");
-    context.put("hdfs.rollInterval", "0");
-    context.put("hdfs.batchSize", "2");
-    context.put("hdfs.idleTimeout", "1");
-    Configurables.configure(sink, context);
-
-    Channel channel = new MemoryChannel();
-    Configurables.configure(channel, context);
-
-    sink.setChannel(channel);
-    sink.start();
-
-    Transaction txn = channel.getTransaction();
-    txn.begin();
-    for (int i = 0; i < 10; i++) {
-      Event event = new SimpleEvent();
-      event.setBody(("test event " + i).getBytes());
-      channel.put(event);
-    }
-    txn.commit();
-    txn.close();
-
-    sink.process();
-    sink.process();
-    Thread.sleep(1101);
-    // previous file should have timed out now
-    // this can throw BucketClosedException(from the bucketWriter having
-    // closed),this is not an issue as the sink will retry and get a fresh
-    // bucketWriter so long as the onClose handler properly removes
-    // bucket writers that were closed.
-    sink.process();
-    sink.process();
-    Thread.sleep(500); // shouldn't be enough for a timeout to occur
-    sink.process();
-    sink.process();
-    sink.stop();
-    FileStatus[] dirStat = fs.listStatus(dirPath);
-    Path[] fList = FileUtil.stat2Paths(dirStat);
-    Assert.assertEquals("Incorrect content of the directory " + StringUtils.join(fList, ","),
-                        2, fList.length);
-    Assert.assertTrue(!fList[0].getName().endsWith(".tmp") &&
-                      !fList[1].getName().endsWith(".tmp"));
-    fs.close();
-  }
-
-  /**
-   * This test simulates what happens when a batch of events is written to a compressed sequence
-   * file (and thus hsync'd to hdfs) but the file is not yet closed.
-   *
-   * When this happens, the data that we wrote should still be readable.
-   */
-  @Test
-  public void testBlockCompressSequenceFileWriterSync() throws IOException, EventDeliveryException {
-    String hdfsPath = testPath + "/sequenceFileWriterSync";
-    FileSystem fs = FileSystem.get(new Configuration());
-    // Since we are reading a partial file we don't want to use checksums
-    fs.setVerifyChecksum(false);
-    fs.setWriteChecksum(false);
-
-    // Compression codecs that don't require native hadoop libraries
-    String [] codecs = {"BZip2Codec", "DeflateCodec"};
-
-    for (String codec : codecs) {
-      sequenceFileWriteAndVerifyEvents(fs, hdfsPath, codec, Collections.singletonList(
-          "single-event"
-      ));
-
-      sequenceFileWriteAndVerifyEvents(fs, hdfsPath, codec, Arrays.asList(
-          "multiple-events-1",
-          "multiple-events-2",
-          "multiple-events-3",
-          "multiple-events-4",
-          "multiple-events-5"
-      ));
-    }
-
-    fs.close();
-  }
-
-  private void sequenceFileWriteAndVerifyEvents(FileSystem fs, String hdfsPath, String codec,
-                                                Collection<String> eventBodies)
-      throws IOException, EventDeliveryException {
-    Path dirPath = new Path(hdfsPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-
-    Context context = new Context();
-    context.put("hdfs.path", hdfsPath);
-    // Ensure the file isn't closed and rolled
-    context.put("hdfs.rollCount", String.valueOf(eventBodies.size() + 1));
-    context.put("hdfs.rollSize", "0");
-    context.put("hdfs.rollInterval", "0");
-    context.put("hdfs.batchSize", "1");
-    context.put("hdfs.fileType", "SequenceFile");
-    context.put("hdfs.codeC", codec);
-    context.put("hdfs.writeFormat", "Writable");
-    Configurables.configure(sink, context);
-
-    Channel channel = new MemoryChannel();
-    Configurables.configure(channel, context);
-
-    sink.setChannel(channel);
-    sink.start();
-
-    for (String eventBody : eventBodies) {
-      Transaction txn = channel.getTransaction();
-      txn.begin();
-
-      Event event = new SimpleEvent();
-      event.setBody(eventBody.getBytes());
-      channel.put(event);
-
-      txn.commit();
-      txn.close();
-
-      sink.process();
-    }
-
-    // Sink is _not_ closed.  The file should remain open but
-    // the data written should be visible to readers via sync + hflush
-    FileStatus[] dirStat = fs.listStatus(dirPath);
-    Path[] paths = FileUtil.stat2Paths(dirStat);
-
-    Assert.assertEquals(1, paths.length);
-
-    SequenceFile.Reader reader =
-        new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.stream(fs.open(paths[0])));
-    LongWritable key = new LongWritable();
-    BytesWritable value = new BytesWritable();
-
-    for (String eventBody : eventBodies) {
-      Assert.assertTrue(reader.next(key, value));
-      Assert.assertArrayEquals(eventBody.getBytes(), value.copyBytes());
-    }
-
-    Assert.assertFalse(reader.next(key, value));
-  }
-
-  private Context getContextForRetryTests() {
-    Context context = new Context();
-
-    context.put("hdfs.path", testPath + "/%{retryHeader}");
-    context.put("hdfs.filePrefix", "test");
-    context.put("hdfs.batchSize", String.valueOf(100));
-    context.put("hdfs.fileType", "DataStream");
-    context.put("hdfs.serializer", "text");
-    context.put("hdfs.closeTries","3");
-    context.put("hdfs.rollCount", "1");
-    context.put("hdfs.retryInterval", "1");
-    return context;
-  }
-
-  @Test
-  public void testBadConfigurationForRetryIntervalZero() throws Exception {
-    Context context = getContextForRetryTests();
-    context.put("hdfs.retryInterval", "0");
-
-    Configurables.configure(sink, context);
-    Assert.assertEquals(1, sink.getTryCount());
-  }
-
-  @Test
-  public void testBadConfigurationForRetryIntervalNegative() throws Exception {
-    Context context = getContextForRetryTests();
-    context.put("hdfs.retryInterval", "-1");
-
-    Configurables.configure(sink, context);
-    Assert.assertEquals(1, sink.getTryCount());
-  }
-
-  @Test
-  public void testBadConfigurationForRetryCountZero() throws Exception {
-    Context context = getContextForRetryTests();
-    context.put("hdfs.closeTries" ,"0");
-
-    Configurables.configure(sink, context);
-    Assert.assertEquals(Integer.MAX_VALUE, sink.getTryCount());
-  }
-
-  @Test
-  public void testBadConfigurationForRetryCountNegative() throws Exception {
-    Context context = getContextForRetryTests();
-    context.put("hdfs.closeTries" ,"-4");
-
-    Configurables.configure(sink, context);
-    Assert.assertEquals(Integer.MAX_VALUE, sink.getTryCount());
-  }
-
-  @Test
-  public void testRetryRename()
-      throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
-    testRetryRename(true);
-    testRetryRename(false);
-  }
-
-  private void testRetryRename(boolean closeSucceed)
-      throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
-    LOG.debug("Starting...");
-    String newPath = testPath + "/retryBucket";
-
-    // clear the test directory
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
-    Path dirPath = new Path(newPath);
-    fs.delete(dirPath, true);
-    fs.mkdirs(dirPath);
-    MockFileSystem mockFs = new MockFileSystem(fs, 6, closeSucceed);
-
-    Context context = getContextForRetryTests();
-    Configurables.configure(sink, context);
-
-    Channel channel = new MemoryChannel();
-    Configurables.configure(channel, context);
-
-    sink.setChannel(channel);
-    sink.setMockFs(mockFs);
-    HDFSWriter hdfsWriter = new MockDataStream(mockFs);
-    hdfsWriter.configure(context);
-    sink.setMockWriter(hdfsWriter);
-    sink.start();
-
-    // push the event batches into channel
-    for (int i = 0; i < 2; i++) {
-      Transaction txn = channel.getTransaction();
-      txn.begin();
-      Map<String, String> hdr = Maps.newHashMap();
-      hdr.put("retryHeader", "v1");
-
-      channel.put(EventBuilder.withBody("random".getBytes(), hdr));
-      txn.commit();
-      txn.close();
-
-      // execute sink to process the events
-      sink.process();
-    }
-    // push the event batches into channel
-    for (int i = 0; i < 2; i++) {
-      Transaction txn = channel.getTransaction();
-      txn.begin();
-      Map<String, String> hdr = Maps.newHashMap();
-      hdr.put("retryHeader", "v2");
-      channel.put(EventBuilder.withBody("random".getBytes(), hdr));
-      txn.commit();
-      txn.close();
-      // execute sink to process the events
-      sink.process();
-    }
-
-    TimeUnit.SECONDS.sleep(5); //Sleep till all retries are done.
-
-    Collection<BucketWriter> writers = sink.getSfWriters().values();
-
-    int totalRenameAttempts = 0;
-    for (BucketWriter writer : writers) {
-      LOG.info("Rename tries = " + writer.renameTries.get());
-      totalRenameAttempts += writer.renameTries.get();
-    }
-    // stop clears the sfWriters map, so we need to compute the
-    // close tries count before stopping the sink.
-    sink.stop();
-    Assert.assertEquals(6, totalRenameAttempts);
-
-  }
-
-  /**
-   * BucketWriter.append() can throw a BucketClosedException when called from
-   * HDFSEventSink.process() due to a race condition between HDFSEventSink.process() and the
-   * BucketWriter's close threads.
-   * This test case tests whether if this happens the newly created BucketWriter will be flushed.
-   * For more details see FLUME-3085
-   */
-  @Test
-  public void testFlushedIfAppendFailedWithBucketClosedException() throws Exception {
-    final Set<BucketWriter> bucketWriters = new HashSet<>();
-    sink = new HDFSEventSink() {
-      @Override
-      BucketWriter initializeBucketWriter(String realPath, String realName, String lookupPath,
-                                          HDFSWriter hdfsWriter, WriterCallback closeCallback) {
-        BucketWriter bw = Mockito.spy(super.initializeBucketWriter(realPath, realName, lookupPath,
-            hdfsWriter, closeCallback));
-        try {
-          // create mock BucketWriters where the first append() succeeds but the
-          // the second call throws a BucketClosedException
-          Mockito.doCallRealMethod()
-              .doThrow(BucketClosedException.class)
-              .when(bw).append(Mockito.any(Event.class));
-        } catch (IOException | InterruptedException e) {
-          Assert.fail("This shouldn't happen, as append() is called during mocking.");
-        }
-        bucketWriters.add(bw);
-        return bw;
-      }
-    };
-
-    Context context = new Context(ImmutableMap.of("hdfs.path", testPath));
-    Configurables.configure(sink, context);
-
-    Channel channel = Mockito.spy(new MemoryChannel());
-    Configurables.configure(channel, new Context());
-
-    final Iterator<Event> events = Iterators.forArray(
-        EventBuilder.withBody("test1".getBytes()), EventBuilder.withBody("test2".getBytes()));
-    Mockito.doAnswer(new Answer() {
-      @Override
-      public Object answer(InvocationOnMock invocation) throws Throwable {
-        return events.hasNext() ? events.next() : null;
-      }
-    }).when(channel).take();
-
-    sink.setChannel(channel);
-    sink.start();
-
-    sink.process();
-
-    // channel.take() should have called 3 times (2 events + 1 null)
-    Mockito.verify(channel, Mockito.times(3)).take();
-
-    FileSystem fs = FileSystem.get(new Configuration());
-    int fileCount = 0;
-    for (RemoteIterator<LocatedFileStatus> i = fs.listFiles(new Path(testPath), false);
-         i.hasNext(); i.next()) {
-      fileCount++;
-    }
-    Assert.assertEquals(2, fileCount);
-
-    Assert.assertEquals(2, bucketWriters.size());
-    // It is expected that flush() method was called exactly once for every BucketWriter
-    for (BucketWriter bw : bucketWriters) {
-      Mockito.verify(bw, Mockito.times(1)).flush();
-    }
-
-    sink.stop();
-  }
-
-  @Test
-  public void testChannelException() {
-    LOG.debug("Starting...");
-    Context context = new Context();
-    context.put("hdfs.path", testPath);
-    context.put("keep-alive", "0");
-    Configurables.configure(sink, context);
-    Channel channel = Mockito.mock(Channel.class);
-    Mockito.when(channel.take()).thenThrow(new ChannelException("dummy"));
-    Mockito.when(channel.getTransaction())
-        .thenReturn(Mockito.mock(BasicTransactionSemantics.class));
-    sink.setChannel(channel);
-    sink.start();
-    try {
-      sink.process();
-    } catch (EventDeliveryException e) {
-      //
-    }
-    sink.stop();
-
-    SinkCounter sc = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
-    Assert.assertEquals(1, sc.getChannelReadFail());
-  }
-
-  @Test
-  public void testEmptyInUseSuffix() {
-    String inUseSuffixConf = "aaaa";
-    Context context = new Context();
-    context.put("hdfs.path", testPath);
-    context.put("hdfs.inUseSuffix", inUseSuffixConf);
-
-    //hdfs.emptyInUseSuffix not defined
-    Configurables.configure(sink, context);
-    String inUseSuffix = (String) Whitebox.getInternalState(sink, "inUseSuffix");
-    Assert.assertEquals(inUseSuffixConf, inUseSuffix);
-
-    context.put("hdfs.emptyInUseSuffix", "true");
-    Configurables.configure(sink, context);
-    inUseSuffix = (String) Whitebox.getInternalState(sink, "inUseSuffix");
-    Assert.assertEquals("", inUseSuffix);
-
-    context.put("hdfs.emptyInUseSuffix", "false");
-    Configurables.configure(sink, context);
-    inUseSuffix = (String) Whitebox.getInternalState(sink, "inUseSuffix");
-    Assert.assertEquals(inUseSuffixConf, inUseSuffix);
-  }
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkDeadlock.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkDeadlock.java
deleted file mode 100644
index 1e1e8642c..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkDeadlock.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.primitives.Longs;
-import org.apache.flume.Channel;
-import org.apache.flume.ChannelSelector;
-import org.apache.flume.Context;
-import org.apache.flume.SinkProcessor;
-import org.apache.flume.SinkRunner;
-import org.apache.flume.channel.ChannelProcessor;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.channel.ReplicatingChannelSelector;
-import org.apache.flume.conf.Configurables;
-import org.apache.flume.sink.DefaultSinkProcessor;
-import org.apache.flume.source.SequenceGeneratorSource;
-
-import java.lang.management.ManagementFactory;
-import java.lang.management.ThreadMXBean;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-public class TestHDFSEventSinkDeadlock {
-
-  public static void main(String... args) {
-    HDFSEventSink sink = new HDFSEventSink();
-    sink.setName("HDFSEventSink");
-
-    Context context = new Context(ImmutableMap.of(
-        "hdfs.path", "file:///tmp/flume-test/bucket-%t",
-        "hdfs.filePrefix", "flumetest",
-        "hdfs.rollInterval", "1",
-        "hdfs.maxOpenFiles", "1",
-        "hdfs.useLocalTimeStamp", "true"));
-    Configurables.configure(sink, context);
-
-    Channel channel = new MemoryChannel();
-    Configurables.configure(channel, new Context());
-
-    final SequenceGeneratorSource source = new SequenceGeneratorSource();
-    Configurables.configure(source, new Context());
-
-    ChannelSelector rcs = new ReplicatingChannelSelector();
-    rcs.setChannels(Collections.singletonList(channel));
-    source.setChannelProcessor(new ChannelProcessor(rcs));
-
-    sink.setChannel(channel);
-
-    channel.start();
-    source.start();
-
-    SinkProcessor sinkProcessor = new DefaultSinkProcessor();
-    sinkProcessor.setSinks(Collections.singletonList(sink));
-    SinkRunner sinkRunner = new SinkRunner();
-    sinkRunner.setSink(sinkProcessor);
-    sinkRunner.start();
-
-    ScheduledExecutorService executor = Executors.newScheduledThreadPool(3);
-
-    executor.execute(new Runnable() {
-      @Override
-      public void run() {
-        int i = 0;
-        while (true) {
-          try {
-            source.process();
-            System.out.println(i++);
-            if (i == 250) {
-              System.out.println("No deadlock found after 250 iterations, exiting");
-              System.exit(0);
-            }
-            Thread.sleep((long) (Math.random() * 100 + 950));
-          } catch (Exception e) {
-            //
-          }
-        }
-      }
-    });
-
-    executor.scheduleAtFixedRate(new Runnable() {
-      @Override
-      public void run() {
-        ThreadMXBean bean = ManagementFactory.getThreadMXBean();
-        long[] threadIds = bean.findDeadlockedThreads();
-        if (threadIds != null) {
-          System.out.println("Deadlocked threads found");
-          printThreadStackTraces(threadIds);
-          System.exit(1);
-        }
-      }
-    }, 0, 1, TimeUnit.SECONDS);
-  }
-
-  private static void printThreadStackTraces(long[] threadIds) {
-    Set<Long> threadIdSet = new HashSet<>(Longs.asList(threadIds));
-    for (Thread th : Thread.getAllStackTraces().keySet()) {
-      if (threadIdSet.contains(th.getId())) {
-        System.out.println("Thread: " + th);
-        for (StackTraceElement e : th.getStackTrace()) {
-          System.out.println("\t" + e);
-        }
-        System.out.println("-----------------------------");
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkOnMiniCluster.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkOnMiniCluster.java
deleted file mode 100644
index 6dfac8a3f..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkOnMiniCluster.java
+++ /dev/null
@@ -1,607 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hdfs;
-
-import com.google.common.base.Charsets;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
-import java.util.zip.GZIPInputStream;
-
-import com.google.common.base.Throwables;
-import org.apache.commons.io.FileUtils;
-import org.apache.flume.Context;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.Transaction;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.event.EventBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Unit tests that exercise HDFSEventSink on an actual instance of HDFS.
- * TODO: figure out how to unit-test Kerberos-secured HDFS.
- */
-public class TestHDFSEventSinkOnMiniCluster {
-
-  private static final Logger logger =
-      LoggerFactory.getLogger(TestHDFSEventSinkOnMiniCluster.class);
-
-  private static final boolean KEEP_DATA = false;
-  private static final String DFS_DIR = "target/test/dfs";
-  private static final String TEST_BUILD_DATA_KEY = "test.build.data";
-
-  private static MiniDFSCluster cluster = null;
-  private static String oldTestBuildDataProp = null;
-
-  @BeforeClass
-  public static void setupClass() throws IOException {
-    // set up data dir for HDFS
-    File dfsDir = new File(DFS_DIR);
-    if (!dfsDir.isDirectory()) {
-      dfsDir.mkdirs();
-    }
-    // save off system prop to restore later
-    oldTestBuildDataProp = System.getProperty(TEST_BUILD_DATA_KEY);
-    System.setProperty(TEST_BUILD_DATA_KEY, DFS_DIR);
-  }
-
-  private static String getNameNodeURL(MiniDFSCluster cluster) {
-    int nnPort = cluster.getNameNode().getNameNodeAddress().getPort();
-    return "hdfs://localhost:" + nnPort;
-  }
-
-  /**
-   * This is a very basic test that writes one event to HDFS and reads it back.
-   */
-  @Test
-  public void simpleHDFSTest() throws EventDeliveryException, IOException {
-    cluster = new MiniDFSCluster(new Configuration(), 1, true, null);
-    cluster.waitActive();
-
-    String outputDir = "/flume/simpleHDFSTest";
-    Path outputDirPath = new Path(outputDir);
-
-    logger.info("Running test with output dir: {}", outputDir);
-
-    FileSystem fs = cluster.getFileSystem();
-    // ensure output directory is empty
-    if (fs.exists(outputDirPath)) {
-      fs.delete(outputDirPath, true);
-    }
-
-    String nnURL = getNameNodeURL(cluster);
-    logger.info("Namenode address: {}", nnURL);
-
-    Context chanCtx = new Context();
-    MemoryChannel channel = new MemoryChannel();
-    channel.setName("simpleHDFSTest-mem-chan");
-    channel.configure(chanCtx);
-    channel.start();
-
-    Context sinkCtx = new Context();
-    sinkCtx.put("hdfs.path", nnURL + outputDir);
-    sinkCtx.put("hdfs.fileType", HDFSWriterFactory.DataStreamType);
-    sinkCtx.put("hdfs.batchSize", Integer.toString(1));
-
-    HDFSEventSink sink = new HDFSEventSink();
-    sink.setName("simpleHDFSTest-hdfs-sink");
-    sink.configure(sinkCtx);
-    sink.setChannel(channel);
-    sink.start();
-
-    // create an event
-    String EVENT_BODY = "yarg!";
-    channel.getTransaction().begin();
-    try {
-      channel.put(EventBuilder.withBody(EVENT_BODY, Charsets.UTF_8));
-      channel.getTransaction().commit();
-    } finally {
-      channel.getTransaction().close();
-    }
-
-    // store event to HDFS
-    sink.process();
-
-    // shut down flume
-    sink.stop();
-    channel.stop();
-
-    // verify that it's in HDFS and that its content is what we say it should be
-    FileStatus[] statuses = fs.listStatus(outputDirPath);
-    Assert.assertNotNull("No files found written to HDFS", statuses);
-    Assert.assertEquals("Only one file expected", 1, statuses.length);
-
-    for (FileStatus status : statuses) {
-      Path filePath = status.getPath();
-      logger.info("Found file on DFS: {}", filePath);
-      FSDataInputStream stream = fs.open(filePath);
-      BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
-      String line = reader.readLine();
-      logger.info("First line in file {}: {}", filePath, line);
-      Assert.assertEquals(EVENT_BODY, line);
-    }
-
-    if (!KEEP_DATA) {
-      fs.delete(outputDirPath, true);
-    }
-
-    cluster.shutdown();
-    cluster = null;
-  }
-
-  /**
-   * Writes two events in GZIP-compressed serialize.
-   */
-  @Test
-  public void simpleHDFSGZipCompressedTest() throws EventDeliveryException, IOException {
-    cluster = new MiniDFSCluster(new Configuration(), 1, true, null);
-    cluster.waitActive();
-
-    String outputDir = "/flume/simpleHDFSGZipCompressedTest";
-    Path outputDirPath = new Path(outputDir);
-
-    logger.info("Running test with output dir: {}", outputDir);
-
-    FileSystem fs = cluster.getFileSystem();
-    // ensure output directory is empty
-    if (fs.exists(outputDirPath)) {
-      fs.delete(outputDirPath, true);
-    }
-
-    String nnURL = getNameNodeURL(cluster);
-    logger.info("Namenode address: {}", nnURL);
-
-    Context chanCtx = new Context();
-    MemoryChannel channel = new MemoryChannel();
-    channel.setName("simpleHDFSTest-mem-chan");
-    channel.configure(chanCtx);
-    channel.start();
-
-    Context sinkCtx = new Context();
-    sinkCtx.put("hdfs.path", nnURL + outputDir);
-    sinkCtx.put("hdfs.fileType", HDFSWriterFactory.CompStreamType);
-    sinkCtx.put("hdfs.batchSize", Integer.toString(1));
-    sinkCtx.put("hdfs.codeC", "gzip");
-
-    HDFSEventSink sink = new HDFSEventSink();
-    sink.setName("simpleHDFSTest-hdfs-sink");
-    sink.configure(sinkCtx);
-    sink.setChannel(channel);
-    sink.start();
-
-    // create an event
-    String EVENT_BODY_1 = "yarg1";
-    String EVENT_BODY_2 = "yarg2";
-    channel.getTransaction().begin();
-    try {
-      channel.put(EventBuilder.withBody(EVENT_BODY_1, Charsets.UTF_8));
-      channel.put(EventBuilder.withBody(EVENT_BODY_2, Charsets.UTF_8));
-      channel.getTransaction().commit();
-    } finally {
-      channel.getTransaction().close();
-    }
-
-    // store event to HDFS
-    sink.process();
-
-    // shut down flume
-    sink.stop();
-    channel.stop();
-
-    // verify that it's in HDFS and that its content is what we say it should be
-    FileStatus[] statuses = fs.listStatus(outputDirPath);
-    Assert.assertNotNull("No files found written to HDFS", statuses);
-    Assert.assertEquals("Only one file expected", 1, statuses.length);
-
-    for (FileStatus status : statuses) {
-      Path filePath = status.getPath();
-      logger.info("Found file on DFS: {}", filePath);
-      FSDataInputStream stream = fs.open(filePath);
-      BufferedReader reader = new BufferedReader(new InputStreamReader(
-          new GZIPInputStream(stream)));
-      String line = reader.readLine();
-      logger.info("First line in file {}: {}", filePath, line);
-      Assert.assertEquals(EVENT_BODY_1, line);
-
-      // The rest of this test is commented-out (will fail) for 2 reasons:
-      //
-      // (1) At the time of this writing, Hadoop has a bug which causes the
-      // non-native gzip implementation to create invalid gzip files when
-      // finish() and resetState() are called. See HADOOP-8522.
-      //
-      // (2) Even if HADOOP-8522 is fixed, the JDK GZipInputStream is unable
-      // to read multi-member (concatenated) gzip files. See this Sun bug:
-      // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4691425
-      //
-      //line = reader.readLine();
-      //logger.info("Second line in file {}: {}", filePath, line);
-      //Assert.assertEquals(EVENT_BODY_2, line);
-    }
-
-    if (!KEEP_DATA) {
-      fs.delete(outputDirPath, true);
-    }
-
-    cluster.shutdown();
-    cluster = null;
-  }
-
-  /**
-   * This is a very basic test that writes one event to HDFS and reads it back.
-   */
-  @Test
-  public void underReplicationTest() throws EventDeliveryException,
-      IOException {
-    Configuration conf = new Configuration();
-    conf.set("dfs.replication", String.valueOf(3));
-    cluster = new MiniDFSCluster(conf, 3, true, null);
-    cluster.waitActive();
-
-    String outputDir = "/flume/underReplicationTest";
-    Path outputDirPath = new Path(outputDir);
-
-    logger.info("Running test with output dir: {}", outputDir);
-
-    FileSystem fs = cluster.getFileSystem();
-    // ensure output directory is empty
-    if (fs.exists(outputDirPath)) {
-      fs.delete(outputDirPath, true);
-    }
-
-    String nnURL = getNameNodeURL(cluster);
-    logger.info("Namenode address: {}", nnURL);
-
-    Context chanCtx = new Context();
-    MemoryChannel channel = new MemoryChannel();
-    channel.setName("simpleHDFSTest-mem-chan");
-    channel.configure(chanCtx);
-    channel.start();
-
-    Context sinkCtx = new Context();
-    sinkCtx.put("hdfs.path", nnURL + outputDir);
-    sinkCtx.put("hdfs.fileType", HDFSWriterFactory.DataStreamType);
-    sinkCtx.put("hdfs.batchSize", Integer.toString(1));
-    sinkCtx.put("hdfs.retryInterval", "10"); //to speed up test
-
-    HDFSEventSink sink = new HDFSEventSink();
-    sink.setName("simpleHDFSTest-hdfs-sink");
-    sink.configure(sinkCtx);
-    sink.setChannel(channel);
-    sink.start();
-
-    // create an event
-    channel.getTransaction().begin();
-    try {
-      channel.put(EventBuilder.withBody("yarg 1", Charsets.UTF_8));
-      channel.put(EventBuilder.withBody("yarg 2", Charsets.UTF_8));
-      channel.put(EventBuilder.withBody("yarg 3", Charsets.UTF_8));
-      channel.put(EventBuilder.withBody("yarg 4", Charsets.UTF_8));
-      channel.put(EventBuilder.withBody("yarg 5", Charsets.UTF_8));
-      channel.put(EventBuilder.withBody("yarg 5", Charsets.UTF_8));
-      channel.getTransaction().commit();
-    } finally {
-      channel.getTransaction().close();
-    }
-
-    // store events to HDFS
-    logger.info("Running process(). Create new file.");
-    sink.process(); // create new file;
-    logger.info("Running process(). Same file.");
-    sink.process();
-
-    // kill a datanode
-    logger.info("Killing datanode #1...");
-    cluster.stopDataNode(0);
-
-    // there is a race here.. the client may or may not notice that the
-    // datanode is dead before it next sync()s.
-    // so, this next call may or may not roll a new file.
-
-    logger.info("Running process(). Create new file? (racy)");
-    sink.process();
-
-    logger.info("Running process(). Create new file.");
-    sink.process();
-
-    logger.info("Running process(). Create new file.");
-    sink.process();
-
-    logger.info("Running process(). Create new file.");
-    sink.process();
-
-    // shut down flume
-    sink.stop();
-    channel.stop();
-
-    // verify that it's in HDFS and that its content is what we say it should be
-    FileStatus[] statuses = fs.listStatus(outputDirPath);
-    Assert.assertNotNull("No files found written to HDFS", statuses);
-
-    for (FileStatus status : statuses) {
-      Path filePath = status.getPath();
-      logger.info("Found file on DFS: {}", filePath);
-      FSDataInputStream stream = fs.open(filePath);
-      BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
-      String line = reader.readLine();
-      logger.info("First line in file {}: {}", filePath, line);
-      Assert.assertTrue(line.startsWith("yarg"));
-    }
-
-    Assert.assertTrue("4 or 5 files expected, found " + statuses.length,
-        statuses.length == 4 || statuses.length == 5);
-    System.out.println("There are " + statuses.length + " files.");
-
-    if (!KEEP_DATA) {
-      fs.delete(outputDirPath, true);
-    }
-
-    cluster.shutdown();
-    cluster = null;
-  }
-
-  /**
-   * This is a very basic test that writes one event to HDFS and reads it back.
-   */
-  @Ignore("This test is flakey and causes tests to fail pretty often.")
-  @Test
-  public void maxUnderReplicationTest() throws EventDeliveryException,
-      IOException {
-    Configuration conf = new Configuration();
-    conf.set("dfs.replication", String.valueOf(3));
-    cluster = new MiniDFSCluster(conf, 3, true, null);
-    cluster.waitActive();
-
-    String outputDir = "/flume/underReplicationTest";
-    Path outputDirPath = new Path(outputDir);
-
-    logger.info("Running test with output dir: {}", outputDir);
-
-    FileSystem fs = cluster.getFileSystem();
-    // ensure output directory is empty
-    if (fs.exists(outputDirPath)) {
-      fs.delete(outputDirPath, true);
-    }
-
-    String nnURL = getNameNodeURL(cluster);
-    logger.info("Namenode address: {}", nnURL);
-
-    Context chanCtx = new Context();
-    MemoryChannel channel = new MemoryChannel();
-    channel.setName("simpleHDFSTest-mem-chan");
-    channel.configure(chanCtx);
-    channel.start();
-
-    Context sinkCtx = new Context();
-    sinkCtx.put("hdfs.path", nnURL + outputDir);
-    sinkCtx.put("hdfs.fileType", HDFSWriterFactory.DataStreamType);
-    sinkCtx.put("hdfs.batchSize", Integer.toString(1));
-
-    HDFSEventSink sink = new HDFSEventSink();
-    sink.setName("simpleHDFSTest-hdfs-sink");
-    sink.configure(sinkCtx);
-    sink.setChannel(channel);
-    sink.start();
-
-    // create an event
-    channel.getTransaction().begin();
-    try {
-      for (int i = 0; i < 50; i++) {
-        channel.put(EventBuilder.withBody("yarg " + i, Charsets.UTF_8));
-      }
-      channel.getTransaction().commit();
-    } finally {
-      channel.getTransaction().close();
-    }
-
-    // store events to HDFS
-    logger.info("Running process(). Create new file.");
-    sink.process(); // create new file;
-    logger.info("Running process(). Same file.");
-    sink.process();
-
-    // kill a datanode
-    logger.info("Killing datanode #1...");
-    cluster.stopDataNode(0);
-
-    // there is a race here.. the client may or may not notice that the
-    // datanode is dead before it next sync()s.
-    // so, this next call may or may not roll a new file.
-
-    logger.info("Running process(). Create new file? (racy)");
-    sink.process();
-
-    for (int i = 3; i < 50; i++) {
-      logger.info("Running process().");
-      sink.process();
-    }
-
-    // shut down flume
-    sink.stop();
-    channel.stop();
-
-    // verify that it's in HDFS and that its content is what we say it should be
-    FileStatus[] statuses = fs.listStatus(outputDirPath);
-    Assert.assertNotNull("No files found written to HDFS", statuses);
-
-    for (FileStatus status : statuses) {
-      Path filePath = status.getPath();
-      logger.info("Found file on DFS: {}", filePath);
-      FSDataInputStream stream = fs.open(filePath);
-      BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
-      String line = reader.readLine();
-      logger.info("First line in file {}: {}", filePath, line);
-      Assert.assertTrue(line.startsWith("yarg"));
-    }
-
-    System.out.println("There are " + statuses.length + " files.");
-    Assert.assertEquals("31 files expected, found " + statuses.length,
-        31, statuses.length);
-
-    if (!KEEP_DATA) {
-      fs.delete(outputDirPath, true);
-    }
-
-    cluster.shutdown();
-    cluster = null;
-  }
-
-  /**
-   * Tests if the lease gets released if the close() call throws IOException.
-   * For more details see https://issues.apache.org/jira/browse/FLUME-3080
-   */
-  @Test
-  public void testLeaseRecoveredIfCloseThrowsIOException() throws Exception {
-    testLeaseRecoveredIfCloseFails(new Callable<Void>() {
-      @Override
-      public Void call() throws Exception {
-        throw new IOException();
-      }
-    });
-  }
-
-  /**
-   * Tests if the lease gets released if the close() call times out.
-   * For more details see https://issues.apache.org/jira/browse/FLUME-3080
-   */
-  @Test
-  public void testLeaseRecoveredIfCloseTimesOut() throws Exception {
-    testLeaseRecoveredIfCloseFails(new Callable<Void>() {
-      @Override
-      public Void call() throws Exception {
-        TimeUnit.SECONDS.sleep(30);
-        return null;
-      }
-    });
-  }
-
-  private void testLeaseRecoveredIfCloseFails(final Callable<?> doThisInClose)
-      throws Exception {
-    cluster = new MiniDFSCluster.Builder(new Configuration()).numDataNodes(1).format(true).build();
-    cluster.waitActive();
-
-    String outputDir = "/flume/leaseRecovery";
-    Path outputDirPath = new Path(outputDir);
-
-    logger.info("Running test with output dir: {}", outputDir);
-
-    FileSystem fs = cluster.getFileSystem();
-    // ensure output directory is empty
-    if (fs.exists(outputDirPath)) {
-      fs.delete(outputDirPath, true);
-    }
-    String nnURL = getNameNodeURL(cluster);
-
-    Context ctx = new Context();
-    MemoryChannel channel = new MemoryChannel();
-    channel.configure(ctx);
-    channel.start();
-
-    ctx.put("hdfs.path", nnURL + outputDir);
-    ctx.put("hdfs.fileType", HDFSWriterFactory.DataStreamType);
-    ctx.put("hdfs.batchSize", Integer.toString(1));
-    ctx.put("hdfs.callTimeout", Integer.toString(1000));
-    ctx.put("hdfs.retryInterval", "10"); //to speed up test
-
-    HDFSWriter hdfsWriter = new HDFSDataStream() {
-      @Override
-      public void close() throws IOException {
-        try {
-          doThisInClose.call();
-        } catch (Throwable e) {
-          Throwables.propagateIfPossible(e, IOException.class);
-          throw new RuntimeException(e);
-        }
-      }
-    };
-    hdfsWriter.configure(ctx);
-
-    HDFSEventSink sink = new HDFSEventSink();
-    sink.configure(ctx);
-    sink.setMockFs(fs);
-    sink.setMockWriter(hdfsWriter);
-    sink.setChannel(channel);
-    sink.start();
-
-    Transaction txn = channel.getTransaction();
-    txn.begin();
-    try {
-      channel.put(EventBuilder.withBody("test", Charsets.UTF_8));
-      txn.commit();
-    } finally {
-      txn.close();
-    }
-
-    sink.process();
-    sink.stop();
-    channel.stop();
-
-    FileStatus[] statuses = fs.listStatus(outputDirPath);
-    Assert.assertEquals(1, statuses.length);
-
-    String filePath = statuses[0].getPath().toUri().getPath();
-
-    // -1 in case that the lease doesn't exist.
-    long leaseRenewalTime = NameNodeAdapter.getLeaseRenewalTime(cluster.getNameNode(), filePath);
-    // wait until the NameNode recovers the lease
-    for (int i = 0; (i < 10) && (leaseRenewalTime != -1L); i++) {
-      TimeUnit.SECONDS.sleep(1);
-      leaseRenewalTime = NameNodeAdapter.getLeaseRenewalTime(cluster.getNameNode(), filePath);
-    }
-
-    // There should be no lease for the given path even if close failed as the BucketWriter
-    // explicitly calls the recoverLease()
-    Assert.assertEquals(-1L, leaseRenewalTime);
-
-    if (!KEEP_DATA) {
-      fs.delete(outputDirPath, true);
-    }
-
-    cluster.shutdown();
-    cluster = null;
-  }
-
-  @AfterClass
-  public static void teardownClass() {
-    // restore system state, if needed
-    if (oldTestBuildDataProp != null) {
-      System.setProperty(TEST_BUILD_DATA_KEY, oldTestBuildDataProp);
-    }
-
-    if (!KEEP_DATA) {
-      FileUtils.deleteQuietly(new File(DFS_DIR));
-    }
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestSequenceFileSerializerFactory.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestSequenceFileSerializerFactory.java
deleted file mode 100644
index 974e857c1..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestSequenceFileSerializerFactory.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import org.apache.flume.Context;
-import org.junit.Test;
-
-import static org.junit.Assert.assertTrue;
-
-public class TestSequenceFileSerializerFactory {
-
-  @Test
-  public void getTextFormatter() {
-    SequenceFileSerializer formatter =
-        SequenceFileSerializerFactory.getSerializer("Text", new Context());
-
-    assertTrue(formatter != null);
-    assertTrue(formatter.getClass().getName(),
-        formatter instanceof HDFSTextSerializer);
-  }
-
-  @Test
-  public void getWritableFormatter() {
-    SequenceFileSerializer formatter =
-        SequenceFileSerializerFactory.getSerializer("Writable", new Context());
-
-    assertTrue(formatter != null);
-    assertTrue(formatter.getClass().getName(),
-        formatter instanceof HDFSWritableSerializer);
-  }
-
-  @Test
-  public void getCustomFormatter() {
-    SequenceFileSerializer formatter = SequenceFileSerializerFactory.getSerializer(
-        "org.apache.flume.sink.hdfs.MyCustomSerializer$Builder", new Context());
-
-    assertTrue(formatter != null);
-    assertTrue(formatter.getClass().getName(),
-        formatter instanceof MyCustomSerializer);
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestUseRawLocalFileSystem.java b/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestUseRawLocalFileSystem.java
deleted file mode 100644
index f3e7d10a7..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestUseRawLocalFileSystem.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flume.sink.hdfs;
-
-import java.io.File;
-import org.apache.commons.io.FileUtils;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.event.EventBuilder;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.GzipCodec;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-
-public class TestUseRawLocalFileSystem {
-
-  private static Logger logger =
-      LoggerFactory.getLogger(TestUseRawLocalFileSystem.class);
-  private Context context;
-
-  private File baseDir;
-  private File testFile;
-  private Event event;
-
-  @Before
-  public void setup() throws Exception {
-    baseDir = Files.createTempDir();
-    testFile = new File(baseDir.getAbsoluteFile(), "test");
-    context = new Context();
-    event = EventBuilder.withBody("test", Charsets.UTF_8);
-  }
-
-  @After
-  public void teardown() throws Exception {
-    FileUtils.deleteQuietly(baseDir);
-  }
-
-  @Test
-  public void testTestFile() throws Exception {
-    String file = testFile.getCanonicalPath();
-    HDFSDataStream stream = new HDFSDataStream();
-    context.put("hdfs.useRawLocalFileSystem", "true");
-    stream.configure(context);
-    stream.open(file);
-    stream.append(event);
-    stream.sync();
-    Assert.assertTrue(testFile.length() > 0);
-  }
-  @Test
-  public void testCompressedFile() throws Exception {
-    String file = testFile.getCanonicalPath();
-    HDFSCompressedDataStream stream = new HDFSCompressedDataStream();
-    context.put("hdfs.useRawLocalFileSystem", "true");
-    stream.configure(context);
-    stream.open(file, new GzipCodec(), CompressionType.RECORD);
-    stream.append(event);
-    stream.sync();
-    Assert.assertTrue(testFile.length() > 0);
-  }
-  @Test
-  public void testSequenceFile() throws Exception {
-    String file = testFile.getCanonicalPath();
-    HDFSSequenceFile stream = new HDFSSequenceFile();
-    context.put("hdfs.useRawLocalFileSystem", "true");
-    stream.configure(context);
-    stream.open(file);
-    stream.append(event);
-    stream.sync();
-    Assert.assertTrue(testFile.length() > 0);
-  }
-
-}
\ No newline at end of file
diff --git a/flume-ng-sinks/flume-hdfs-sink/src/test/resources/log4j2.xml b/flume-ng-sinks/flume-hdfs-sink/src/test/resources/log4j2.xml
deleted file mode 100644
index 91a23f8fa..000000000
--- a/flume-ng-sinks/flume-hdfs-sink/src/test/resources/log4j2.xml
+++ /dev/null
@@ -1,33 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements.  See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-<Configuration status="OFF">
-  <Appenders>
-    <Console name="Console" target="SYSTEM_OUT">
-      <PatternLayout pattern="%d (%t) [%p - %l] %m%n" />
-    </Console>
-  </Appenders>
-  <Loggers>
-    <Logger name="org.apache.flume" level="DEBUG"/>
-    <Logger name="org.apache.hadoop" level="WARN"/>
-    <Logger name="org.mortbay" level="WARN"/>
-    <Root level="INFO">
-      <AppenderRef ref="Console" />
-    </Root>
-  </Loggers>
-</Configuration>
\ No newline at end of file
diff --git a/flume-ng-sinks/flume-hive-sink/pom.xml b/flume-ng-sinks/flume-hive-sink/pom.xml
deleted file mode 100644
index 0047b9912..000000000
--- a/flume-ng-sinks/flume-hive-sink/pom.xml
+++ /dev/null
@@ -1,149 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements.  See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-  <modelVersion>4.0.0</modelVersion>
-
-  <parent>
-    <groupId>org.apache.flume</groupId>
-    <artifactId>flume-ng-sinks</artifactId>
-    <version>1.11.1-SNAPSHOT</version>
-  </parent>
-
-  <properties>
-    <!-- TODO fix spotbugs/pmd violations -->
-    <spotbugs.maxAllowedViolations>6</spotbugs.maxAllowedViolations>
-    <pmd.maxAllowedViolations>7</pmd.maxAllowedViolations>
-    <module.name>org.apache.flume.sink.hive</module.name>
-  </properties>
-
-  <groupId>org.apache.flume.flume-ng-sinks</groupId>
-  <artifactId>flume-hive-sink</artifactId>
-  <name>Flume NG Hive Sink</name>
-
-  <repositories>
-    <!-- Our current hive version needs a calcite version that has dependencies not available in Maven Central.
-         The pom from calcite gives the HTTP version of this repo address, which gets ignored by current maven.
-         The original conjars repo is dead.  Use the read-only mirror.
-      -->
-    <repository>
-      <releases>
-        <enabled>true</enabled>
-        <updatePolicy>always</updatePolicy>
-        <checksumPolicy>warn</checksumPolicy>
-      </releases>
-      <id>conjars</id>
-      <name>Conjars</name>
-      <url>https://conjars.wensel.net/repo/</url>
-      <layout>default</layout>
-    </repository>
-  </repositories>
-  
-  <dependencyManagement>
-    <dependencies>
-      <dependency>
-        <groupId>org.glassfish</groupId>
-        <artifactId>javax.el</artifactId>
-        <version>3.0.1-b12</version>
-      </dependency>
-    </dependencies>
-  </dependencyManagement>
-
-  <dependencies>
-
-    <dependency>
-      <groupId>org.apache.flume</groupId>
-      <artifactId>flume-ng-sdk</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.flume</groupId>
-      <artifactId>flume-ng-configuration</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.flume</groupId>
-      <artifactId>flume-ng-core</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-api</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-slf4j-impl</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-1.2-api</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hive.hcatalog</groupId>
-      <artifactId>hive-hcatalog-streaming</artifactId>
-      <scope>provided</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hive.hcatalog</groupId>
-      <artifactId>hive-hcatalog-core</artifactId>
-      <scope>provided</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hive</groupId>
-      <artifactId>hive-cli</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <scope>test</scope>
-      <optional>true</optional>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-mapreduce-client-core</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <!-- end temporary -->
-
-    <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-all</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-  </dependencies>
-
-</project>
diff --git a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/Config.java b/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/Config.java
deleted file mode 100644
index b2d258227..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/Config.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hive;
-
-public class Config {
-  public static final String HIVE_METASTORE = "hive.metastore";
-  public static final String HIVE_DATABASE = "hive.database";
-  public static final String HIVE_TABLE = "hive.table";
-  public static final String HIVE_PARTITION = "hive.partition";
-  public static final String HIVE_TXNS_PER_BATCH_ASK = "hive.txnsPerBatchAsk";
-  public static final String BATCH_SIZE = "batchSize";
-  public static final String IDLE_TIMEOUT = "idleTimeout";
-  public static final String CALL_TIMEOUT = "callTimeout";
-  public static final String HEART_BEAT_INTERVAL = "heartBeatInterval";
-  public static final String MAX_OPEN_CONNECTIONS = "maxOpenConnections";
-  public static final String USE_LOCAL_TIME_STAMP = "useLocalTimeStamp";
-  public static final String TIME_ZONE = "timeZone";
-  public static final String ROUND_UNIT = "roundUnit";
-  public static final String ROUND = "round";
-  public static final String HOUR = "hour";
-  public static final String MINUTE = "minute";
-  public static final String SECOND = "second";
-  public static final String ROUND_VALUE = "roundValue";
-  public static final String SERIALIZER = "serializer";
-}
diff --git a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveDelimitedTextSerializer.java b/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveDelimitedTextSerializer.java
deleted file mode 100644
index 59520e7d8..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveDelimitedTextSerializer.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hive;
-
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.hive.hcatalog.streaming.DelimitedInputWriter;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.apache.hive.hcatalog.streaming.RecordWriter;
-import org.apache.hive.hcatalog.streaming.StreamingException;
-import org.apache.hive.hcatalog.streaming.TransactionBatch;
-
-import java.io.IOException;
-import java.util.Collection;
-
-/** Forwards the incoming event body to Hive unmodified
- * Sets up the delimiter and the field to column mapping
- */
-public class HiveDelimitedTextSerializer implements HiveEventSerializer  {
-  public static final String ALIAS = "DELIMITED";
-
-  public static final String defaultDelimiter = ",";
-  public static final String SERIALIZER_DELIMITER = "serializer.delimiter";
-  public static final String SERIALIZER_FIELDNAMES = "serializer.fieldnames";
-  public static final String SERIALIZER_SERDE_SEPARATOR = "serializer.serdeSeparator";
-
-  private String delimiter;
-  private String[] fieldToColMapping = null;
-  private Character serdeSeparator = null;
-
-  @Override
-  public void write(TransactionBatch txnBatch, Event e)
-          throws StreamingException, IOException, InterruptedException {
-    txnBatch.write(e.getBody());
-  }
-
-  @Override
-  public void write(TransactionBatch txnBatch, Collection<byte[]> events)
-          throws StreamingException, IOException, InterruptedException {
-    txnBatch.write(events);
-  }
-
-
-  @Override
-  public RecordWriter createRecordWriter(HiveEndPoint endPoint)
-      throws StreamingException, IOException, ClassNotFoundException {
-    if (serdeSeparator == null) {
-      return new DelimitedInputWriter(fieldToColMapping, delimiter, endPoint);
-    }
-    return new DelimitedInputWriter(fieldToColMapping, delimiter, endPoint, null, serdeSeparator);
-  }
-
-  @Override
-  public void configure(Context context) {
-    delimiter = parseDelimiterSpec(
-            context.getString(SERIALIZER_DELIMITER, defaultDelimiter) );
-    String fieldNames = context.getString(SERIALIZER_FIELDNAMES);
-    if (fieldNames == null) {
-      throw new IllegalArgumentException("serializer.fieldnames is not specified " +
-              "for serializer " + this.getClass().getName() );
-    }
-    String serdeSeparatorStr = context.getString(SERIALIZER_SERDE_SEPARATOR);
-    this.serdeSeparator = parseSerdeSeparatorSpec(serdeSeparatorStr);
-
-    // split, but preserve empty fields (-1)
-    fieldToColMapping = fieldNames.trim().split(",",-1);
-  }
-
-  // if delimiter is a double quoted like "\t", drop quotes
-  private static String parseDelimiterSpec(String delimiter) {
-    if (delimiter == null) {
-      return null;
-    }
-    if (delimiter.charAt(0) == '"'  &&
-        delimiter.charAt(delimiter.length() - 1) == '"') {
-      return delimiter.substring(1,delimiter.length() - 1);
-    }
-    return delimiter;
-  }
-
-  // if delimiter is a single quoted character like '\t', drop quotes
-  private static  Character parseSerdeSeparatorSpec(String separatorStr) {
-    if (separatorStr == null) {
-      return null;
-    }
-    if (separatorStr.length() == 1) {
-      return separatorStr.charAt(0);
-    }
-    if (separatorStr.length() == 3    &&
-        separatorStr.charAt(2) == '\''  &&
-        separatorStr.charAt(separatorStr.length() - 1) == '\'') {
-      return separatorStr.charAt(1);
-    }
-
-    throw new IllegalArgumentException("serializer.serdeSeparator spec is invalid " +
-            "for " + ALIAS + " serializer " );
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveEventSerializer.java b/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveEventSerializer.java
deleted file mode 100644
index 7ed2c8276..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveEventSerializer.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hive;
-
-import org.apache.flume.Event;
-import org.apache.flume.conf.Configurable;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.apache.hive.hcatalog.streaming.RecordWriter;
-import org.apache.hive.hcatalog.streaming.StreamingException;
-import org.apache.hive.hcatalog.streaming.TransactionBatch;
-
-import java.io.IOException;
-import java.util.Collection;
-
-public interface HiveEventSerializer extends Configurable {
-  public void write(TransactionBatch batch, Event e)
-          throws StreamingException, IOException, InterruptedException;
-
-  public void write(TransactionBatch txnBatch, Collection<byte[]> events)
-          throws StreamingException, IOException, InterruptedException;
-
-  RecordWriter createRecordWriter(HiveEndPoint endPoint)
-          throws StreamingException, IOException, ClassNotFoundException;
-
-}
diff --git a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveJsonSerializer.java b/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveJsonSerializer.java
deleted file mode 100644
index 0311a5b7d..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveJsonSerializer.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hive;
-
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.apache.hive.hcatalog.streaming.RecordWriter;
-import org.apache.hive.hcatalog.streaming.StreamingException;
-import org.apache.hive.hcatalog.streaming.StrictJsonWriter;
-import org.apache.hive.hcatalog.streaming.TransactionBatch;
-
-import java.io.IOException;
-import java.util.Collection;
-
-/** Forwards the incoming event body to Hive unmodified
- * Sets up the delimiter and the field to column mapping
- */
-
-public class HiveJsonSerializer implements HiveEventSerializer  {
-  public static final String ALIAS = "JSON";
-
-  @Override
-  public void write(TransactionBatch txnBatch, Event e)
-          throws StreamingException, IOException, InterruptedException {
-    txnBatch.write(e.getBody());
-  }
-
-  @Override
-  public void write(TransactionBatch txnBatch, Collection<byte[]> events)
-          throws StreamingException, IOException, InterruptedException {
-    txnBatch.write(events);
-  }
-
-  @Override
-  public RecordWriter createRecordWriter(HiveEndPoint endPoint)
-          throws StreamingException, IOException, ClassNotFoundException {
-    return new StrictJsonWriter(endPoint);
-  }
-
-  @Override
-  public void configure(Context context) {
-    return;
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveSink.java b/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveSink.java
deleted file mode 100644
index ec1fe621c..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveSink.java
+++ /dev/null
@@ -1,528 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hive;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.flume.Channel;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.Transaction;
-import org.apache.flume.conf.BatchSizeSupported;
-import org.apache.flume.conf.Configurable;
-import org.apache.flume.formatter.output.BucketPath;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.flume.sink.AbstractSink;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Calendar;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TimeZone;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class HiveSink extends AbstractSink implements Configurable, BatchSizeSupported {
-
-  private static final Logger LOG = LoggerFactory.getLogger(HiveSink.class);
-
-  private static final int DEFAULT_MAXOPENCONNECTIONS = 500;
-  private static final int DEFAULT_TXNSPERBATCH = 100;
-  private static final int DEFAULT_BATCHSIZE = 15000;
-  private static final int DEFAULT_CALLTIMEOUT = 10000;
-  private static final int DEFAULT_IDLETIMEOUT = 0;
-  private static final int DEFAULT_HEARTBEATINTERVAL = 240; // seconds
-
-  private Map<HiveEndPoint, HiveWriter> allWriters;
-
-  private SinkCounter sinkCounter;
-  private volatile int idleTimeout;
-  private String metaStoreUri;
-  private String proxyUser;
-  private String database;
-  private String table;
-  private List<String> partitionVals;
-  private Integer txnsPerBatchAsk;
-  private Integer batchSize;
-  private Integer maxOpenConnections;
-  private boolean autoCreatePartitions;
-  private String serializerType;
-  private HiveEventSerializer serializer;
-
-  /**
-   * Default timeout for blocking I/O calls in HiveWriter
-   */
-  private Integer callTimeout;
-  private Integer heartBeatInterval;
-
-  private ExecutorService callTimeoutPool;
-
-  private boolean useLocalTime;
-  private TimeZone timeZone;
-  private boolean needRounding;
-  private int roundUnit;
-  private Integer roundValue;
-
-  private Timer heartBeatTimer = new Timer();
-  private AtomicBoolean timeToSendHeartBeat = new AtomicBoolean(false);
-
-  @VisibleForTesting
-  Map<HiveEndPoint, HiveWriter> getAllWriters() {
-    return allWriters;
-  }
-
-  // read configuration and setup thresholds
-  @Override
-  public void configure(Context context) {
-
-    metaStoreUri = context.getString(Config.HIVE_METASTORE);
-    if (metaStoreUri == null) {
-      throw new IllegalArgumentException(Config.HIVE_METASTORE + " config setting is not " +
-              "specified for sink " + getName());
-    }
-    if (metaStoreUri.equalsIgnoreCase("null")) { // for testing support
-      metaStoreUri = null;
-    }
-    proxyUser = null; // context.getString("hive.proxyUser"); not supported by hive api yet
-    database = context.getString(Config.HIVE_DATABASE);
-    if (database == null) {
-      throw new IllegalArgumentException(Config.HIVE_DATABASE + " config setting is not " +
-            "specified for sink " + getName());
-    }
-    table = context.getString(Config.HIVE_TABLE);
-    if (table == null) {
-      throw new IllegalArgumentException(Config.HIVE_TABLE + " config setting is not " +
-              "specified for sink " + getName());
-    }
-
-    String partitions = context.getString(Config.HIVE_PARTITION);
-    if (partitions != null) {
-      partitionVals = Arrays.asList(partitions.split(","));
-    }
-
-
-    txnsPerBatchAsk = context.getInteger(Config.HIVE_TXNS_PER_BATCH_ASK, DEFAULT_TXNSPERBATCH);
-    if (txnsPerBatchAsk < 0) {
-      LOG.warn(getName() + ". hive.txnsPerBatchAsk must be  positive number. Defaulting to "
-              + DEFAULT_TXNSPERBATCH);
-      txnsPerBatchAsk = DEFAULT_TXNSPERBATCH;
-    }
-    batchSize = context.getInteger(Config.BATCH_SIZE, DEFAULT_BATCHSIZE);
-    if (batchSize < 0) {
-      LOG.warn(getName() + ". batchSize must be  positive number. Defaulting to "
-              + DEFAULT_BATCHSIZE);
-      batchSize = DEFAULT_BATCHSIZE;
-    }
-    idleTimeout = context.getInteger(Config.IDLE_TIMEOUT, DEFAULT_IDLETIMEOUT);
-    if (idleTimeout < 0) {
-      LOG.warn(getName() + ". idleTimeout must be  positive number. Defaulting to "
-              + DEFAULT_IDLETIMEOUT);
-      idleTimeout = DEFAULT_IDLETIMEOUT;
-    }
-    callTimeout = context.getInteger(Config.CALL_TIMEOUT, DEFAULT_CALLTIMEOUT);
-    if (callTimeout < 0) {
-      LOG.warn(getName() + ". callTimeout must be  positive number. Defaulting to "
-              + DEFAULT_CALLTIMEOUT);
-      callTimeout = DEFAULT_CALLTIMEOUT;
-    }
-
-    heartBeatInterval = context.getInteger(Config.HEART_BEAT_INTERVAL, DEFAULT_HEARTBEATINTERVAL);
-    if (heartBeatInterval < 0) {
-      LOG.warn(getName() + ". heartBeatInterval must be  positive number. Defaulting to "
-              + DEFAULT_HEARTBEATINTERVAL);
-      heartBeatInterval = DEFAULT_HEARTBEATINTERVAL;
-    }
-    maxOpenConnections = context.getInteger(Config.MAX_OPEN_CONNECTIONS,
-                                            DEFAULT_MAXOPENCONNECTIONS);
-    autoCreatePartitions =  context.getBoolean("autoCreatePartitions", true);
-
-    // Timestamp processing
-    useLocalTime = context.getBoolean(Config.USE_LOCAL_TIME_STAMP, false);
-
-    String tzName = context.getString(Config.TIME_ZONE);
-    timeZone = (tzName == null) ? null : TimeZone.getTimeZone(tzName);
-    needRounding = context.getBoolean(Config.ROUND, false);
-
-    String unit = context.getString(Config.ROUND_UNIT, Config.MINUTE);
-    if (unit.equalsIgnoreCase(Config.HOUR)) {
-      this.roundUnit = Calendar.HOUR_OF_DAY;
-    } else if (unit.equalsIgnoreCase(Config.MINUTE)) {
-      this.roundUnit = Calendar.MINUTE;
-    } else if (unit.equalsIgnoreCase(Config.SECOND)) {
-      this.roundUnit = Calendar.SECOND;
-    } else {
-      LOG.warn(getName() + ". Rounding unit is not valid, please set one of " +
-              "minute, hour or second. Rounding will be disabled");
-      needRounding = false;
-    }
-    this.roundValue = context.getInteger(Config.ROUND_VALUE, 1);
-    if (roundUnit == Calendar.SECOND || roundUnit == Calendar.MINUTE) {
-      Preconditions.checkArgument(roundValue > 0 && roundValue <= 60,
-              "Round value must be > 0 and <= 60");
-    } else if (roundUnit == Calendar.HOUR_OF_DAY) {
-      Preconditions.checkArgument(roundValue > 0 && roundValue <= 24,
-              "Round value must be > 0 and <= 24");
-    }
-
-    // Serializer
-    serializerType = context.getString(Config.SERIALIZER, "");
-    if (serializerType.isEmpty()) {
-      throw new IllegalArgumentException("serializer config setting is not " +
-              "specified for sink " + getName());
-    }
-
-    serializer = createSerializer(serializerType);
-    serializer.configure(context);
-
-    Preconditions.checkArgument(batchSize > 0, "batchSize must be greater than 0");
-
-    if (sinkCounter == null) {
-      sinkCounter = new SinkCounter(getName());
-    }
-  }
-
-  @VisibleForTesting
-  protected SinkCounter getCounter() {
-    return sinkCounter;
-  }
-  private HiveEventSerializer createSerializer(String serializerName)  {
-    if (serializerName.compareToIgnoreCase(HiveDelimitedTextSerializer.ALIAS) == 0 ||
-        serializerName.compareTo(HiveDelimitedTextSerializer.class.getName()) == 0) {
-      return new HiveDelimitedTextSerializer();
-    } else if (serializerName.compareToIgnoreCase(HiveJsonSerializer.ALIAS) == 0 ||
-            serializerName.compareTo(HiveJsonSerializer.class.getName()) == 0) {
-      return new HiveJsonSerializer();
-    }
-
-    try {
-      return (HiveEventSerializer) Class.forName(serializerName).newInstance();
-    } catch (Exception e) {
-      throw new IllegalArgumentException("Unable to instantiate serializer: " + serializerName
-              + " on sink: " + getName(), e);
-    }
-  }
-
-
-  /**
-   * Pull events out of channel, find corresponding HiveWriter and write to it.
-   * Take at most batchSize events per Transaction. <br/>
-   * This method is not thread safe.
-   */
-  public Status process() throws EventDeliveryException {
-    // writers used in this Txn
-
-    Channel channel = getChannel();
-    Transaction transaction = channel.getTransaction();
-    transaction.begin();
-    boolean success = false;
-    try {
-      // 1 Enable Heart Beats
-      if (timeToSendHeartBeat.compareAndSet(true, false)) {
-        enableHeartBeatOnAllWriters();
-      }
-
-      // 2 Drain Batch
-      int txnEventCount = drainOneBatch(channel);
-      transaction.commit();
-      success = true;
-
-      // 3 Update Counters
-      if (txnEventCount < 1) {
-        return Status.BACKOFF;
-      } else {
-        return Status.READY;
-      }
-    } catch (InterruptedException err) {
-      LOG.warn(getName() + ": Thread was interrupted.", err);
-      return Status.BACKOFF;
-    } catch (Exception e) {
-      sinkCounter.incrementEventWriteOrChannelFail(e);
-      throw new EventDeliveryException(e);
-    } finally {
-      if (!success) {
-        transaction.rollback();
-      }
-      transaction.close();
-    }
-  }
-
-  // Drains one batch of events from Channel into Hive
-  private int drainOneBatch(Channel channel)
-          throws HiveWriter.Failure, InterruptedException {
-    int txnEventCount = 0;
-    try {
-      Map<HiveEndPoint,HiveWriter> activeWriters = Maps.newHashMap();
-      for (; txnEventCount < batchSize; ++txnEventCount) {
-        // 0) Read event from Channel
-        Event event = channel.take();
-        if (event == null) {
-          break;
-        }
-
-        //1) Create end point by substituting place holders
-        HiveEndPoint endPoint = makeEndPoint(metaStoreUri, database, table,
-                partitionVals, event.getHeaders(), timeZone,
-                needRounding, roundUnit, roundValue, useLocalTime);
-
-        //2) Create or reuse Writer
-        HiveWriter writer = getOrCreateWriter(activeWriters, endPoint);
-
-        //3) Write
-        LOG.debug("{} : Writing event to {}", getName(), endPoint);
-        writer.write(event);
-
-      } // for
-
-      //4) Update counters
-      if (txnEventCount == 0) {
-        sinkCounter.incrementBatchEmptyCount();
-      } else if (txnEventCount == batchSize) {
-        sinkCounter.incrementBatchCompleteCount();
-      } else {
-        sinkCounter.incrementBatchUnderflowCount();
-      }
-      sinkCounter.addToEventDrainAttemptCount(txnEventCount);
-
-
-      // 5) Flush all Writers
-      for (HiveWriter writer : activeWriters.values()) {
-        writer.flush(true);
-      }
-
-      sinkCounter.addToEventDrainSuccessCount(txnEventCount);
-      return txnEventCount;
-    } catch (HiveWriter.Failure e) {
-      // in case of error we close all TxnBatches to start clean next time
-      LOG.warn(getName() + " : " + e.getMessage(), e);
-      abortAllWriters();
-      closeAllWriters();
-      throw e;
-    }
-  }
-
-  private void enableHeartBeatOnAllWriters() {
-    for (HiveWriter writer : allWriters.values()) {
-      writer.setHearbeatNeeded();
-    }
-  }
-
-  private HiveWriter getOrCreateWriter(Map<HiveEndPoint, HiveWriter> activeWriters,
-                                       HiveEndPoint endPoint)
-          throws HiveWriter.ConnectException, InterruptedException {
-    try {
-      HiveWriter writer = allWriters.get( endPoint );
-      if (writer == null) {
-        LOG.info(getName() + ": Creating Writer to Hive end point : " + endPoint);
-        writer = new HiveWriter(endPoint, txnsPerBatchAsk, autoCreatePartitions,
-                callTimeout, callTimeoutPool, proxyUser, serializer, sinkCounter);
-
-        sinkCounter.incrementConnectionCreatedCount();
-        if (allWriters.size() > maxOpenConnections) {
-          int retired = closeIdleWriters();
-          if (retired == 0) {
-            closeEldestWriter();
-          }
-        }
-        allWriters.put(endPoint, writer);
-        activeWriters.put(endPoint, writer);
-      } else {
-        if (activeWriters.get(endPoint) == null)  {
-          activeWriters.put(endPoint,writer);
-        }
-      }
-      return writer;
-    } catch (HiveWriter.ConnectException e) {
-      sinkCounter.incrementConnectionFailedCount();
-      throw e;
-    }
-
-  }
-
-  private HiveEndPoint makeEndPoint(String metaStoreUri, String database, String table,
-                                    List<String> partVals, Map<String, String> headers,
-                                    TimeZone timeZone, boolean needRounding,
-                                    int roundUnit, Integer roundValue,
-                                    boolean useLocalTime)  {
-    if (partVals == null) {
-      return new HiveEndPoint(metaStoreUri, database, table, null);
-    }
-
-    ArrayList<String> realPartVals = Lists.newArrayList();
-    for (String partVal : partVals) {
-      realPartVals.add(BucketPath.escapeString(partVal, headers, timeZone,
-              needRounding, roundUnit, roundValue, useLocalTime));
-    }
-    return new HiveEndPoint(metaStoreUri, database, table, realPartVals);
-  }
-
-  /**
-   * Locate writer that has not been used for longest time and retire it
-   */
-  private void closeEldestWriter() throws InterruptedException {
-    long oldestTimeStamp = System.currentTimeMillis();
-    HiveEndPoint eldest = null;
-    for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
-      if (entry.getValue().getLastUsed() < oldestTimeStamp) {
-        eldest = entry.getKey();
-        oldestTimeStamp = entry.getValue().getLastUsed();
-      }
-    }
-
-    try {
-      sinkCounter.incrementConnectionCreatedCount();
-      LOG.info(getName() + ": Closing least used Writer to Hive EndPoint : " + eldest);
-      allWriters.remove(eldest).close();
-    } catch (InterruptedException e) {
-      LOG.warn(getName() + ": Interrupted when attempting to close writer for end point: "
-              + eldest, e);
-      throw e;
-    }
-  }
-
-  /**
-   * Locate all writers past idle timeout and retire them
-   * @return number of writers retired
-   */
-  private int closeIdleWriters() throws InterruptedException {
-    int count = 0;
-    long now = System.currentTimeMillis();
-    ArrayList<HiveEndPoint> retirees = Lists.newArrayList();
-
-    //1) Find retirement candidates
-    for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
-      if (now - entry.getValue().getLastUsed() > idleTimeout) {
-        ++count;
-        retirees.add(entry.getKey());
-      }
-    }
-    //2) Retire them
-    for (HiveEndPoint ep : retirees) {
-      sinkCounter.incrementConnectionClosedCount();
-      LOG.info(getName() + ": Closing idle Writer to Hive end point : {}", ep);
-      allWriters.remove(ep).close();
-    }
-    return count;
-  }
-
-  /**
-   * Closes all writers and remove them from cache
-   * @return number of writers retired
-   */
-  private void closeAllWriters() throws InterruptedException {
-    //1) Retire writers
-    for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
-      entry.getValue().close();
-    }
-
-    //2) Clear cache
-    allWriters.clear();
-  }
-
-  /**
-   * Abort current Txn on all writers
-   * @return number of writers retired
-   */
-  private void abortAllWriters() throws InterruptedException {
-    for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
-      entry.getValue().abort();
-    }
-  }
-
-  @Override
-  public void stop() {
-    // do not constrain close() calls with a timeout
-    for (Entry<HiveEndPoint, HiveWriter> entry : allWriters.entrySet()) {
-      try {
-        HiveWriter w = entry.getValue();
-        w.close();
-      } catch (InterruptedException ex) {
-        Thread.currentThread().interrupt();
-      }
-    }
-
-    // shut down all thread pools
-    callTimeoutPool.shutdown();
-    try {
-      while (callTimeoutPool.isTerminated() == false) {
-        callTimeoutPool.awaitTermination(
-              Math.max(DEFAULT_CALLTIMEOUT, callTimeout), TimeUnit.MILLISECONDS);
-      }
-    } catch (InterruptedException ex) {
-      LOG.warn(getName() + ":Shutdown interrupted on " + callTimeoutPool, ex);
-    }
-
-    callTimeoutPool = null;
-    allWriters.clear();
-    allWriters = null;
-    sinkCounter.stop();
-    super.stop();
-    LOG.info("Hive Sink {} stopped", getName() );
-  }
-
-  @Override
-  public void start() {
-    String timeoutName = "hive-" + getName() + "-call-runner-%d";
-    // call timeout pool needs only 1 thd as sink is effectively single threaded
-    callTimeoutPool = Executors.newFixedThreadPool(1,
-            new ThreadFactoryBuilder().setNameFormat(timeoutName).build());
-
-    this.allWriters = Maps.newHashMap();
-    sinkCounter.start();
-    super.start();
-    setupHeartBeatTimer();
-    LOG.info(getName() + ": Hive Sink {} started", getName() );
-  }
-
-  private void setupHeartBeatTimer() {
-    if (heartBeatInterval > 0) {
-      heartBeatTimer.schedule(new TimerTask() {
-        @Override
-        public void run() {
-          timeToSendHeartBeat.set(true);
-          setupHeartBeatTimer();
-        }
-      }, heartBeatInterval * 1000);
-    }
-  }
-
-  @Override
-  public long getBatchSize() {
-    return batchSize;
-  }
-
-  @Override
-  public String toString() {
-    return "{ Sink type:" + getClass().getSimpleName() + ", name:" + getName() +
-            " }";
-  }
-
-}
diff --git a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveWriter.java b/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveWriter.java
deleted file mode 100644
index 7106696a1..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/main/java/org/apache/flume/sink/hive/HiveWriter.java
+++ /dev/null
@@ -1,513 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flume.sink.hive;
-
-import org.apache.flume.Event;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.apache.hive.hcatalog.streaming.RecordWriter;
-import org.apache.hive.hcatalog.streaming.SerializationError;
-import org.apache.hive.hcatalog.streaming.StreamingConnection;
-import org.apache.hive.hcatalog.streaming.StreamingException;
-import org.apache.hive.hcatalog.streaming.StreamingIOFailure;
-import org.apache.hive.hcatalog.streaming.TransactionBatch;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * Internal API intended for HiveSink use.
- */
-class HiveWriter {
-
-  private static final Logger LOG = LoggerFactory.getLogger(HiveWriter.class);
-
-  private final HiveEndPoint endPoint;
-  private HiveEventSerializer serializer;
-  private final StreamingConnection connection;
-  private final int txnsPerBatch;
-  private final RecordWriter recordWriter;
-  private TransactionBatch txnBatch;
-
-  private final ExecutorService callTimeoutPool;
-
-  private final long callTimeout;
-
-  private long lastUsed; // time of last flush on this writer
-
-  private SinkCounter sinkCounter;
-  private int batchCounter;
-  private long eventCounter;
-  private long processSize;
-
-  protected boolean closed; // flag indicating HiveWriter was closed
-  private boolean autoCreatePartitions;
-
-  private boolean hearbeatNeeded = false;
-
-  private final int writeBatchSz = 1000;
-  private ArrayList<Event> batch = new ArrayList<Event>(writeBatchSz);
-
-  HiveWriter(HiveEndPoint endPoint, int txnsPerBatch,
-             boolean autoCreatePartitions, long callTimeout,
-             ExecutorService callTimeoutPool, String hiveUser,
-             HiveEventSerializer serializer, SinkCounter sinkCounter)
-      throws ConnectException, InterruptedException {
-    try {
-      this.autoCreatePartitions = autoCreatePartitions;
-      this.sinkCounter = sinkCounter;
-      this.callTimeout = callTimeout;
-      this.callTimeoutPool = callTimeoutPool;
-      this.endPoint = endPoint;
-      this.connection = newConnection(hiveUser);
-      this.txnsPerBatch = txnsPerBatch;
-      this.serializer = serializer;
-      this.recordWriter = serializer.createRecordWriter(endPoint);
-      this.txnBatch = nextTxnBatch(recordWriter);
-      this.txnBatch.beginNextTransaction();
-      this.closed = false;
-      this.lastUsed = System.currentTimeMillis();
-    } catch (InterruptedException e) {
-      throw e;
-    } catch (RuntimeException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new ConnectException(endPoint, e);
-    }
-  }
-
-  @Override
-  public String toString() {
-    return endPoint.toString();
-  }
-
-  /**
-   * Clear the class counters
-   */
-  private void resetCounters() {
-    eventCounter = 0;
-    processSize = 0;
-    batchCounter = 0;
-  }
-
-  void setHearbeatNeeded() {
-    hearbeatNeeded = true;
-  }
-
-  public int getRemainingTxns() {
-    return txnBatch.remainingTransactions();
-  }
-
-
-  /**
-   * Write data, update stats
-   * @param event
-   * @throws WriteException - other streaming io error
-   * @throws InterruptedException
-   */
-  public synchronized void write(final Event event)
-      throws WriteException, InterruptedException {
-    if (closed) {
-      throw new IllegalStateException("Writer closed. Cannot write to : " + endPoint);
-    }
-
-    batch.add(event);
-    if (batch.size() == writeBatchSz) {
-      // write the event
-      writeEventBatchToSerializer();
-    }
-
-    // Update Statistics
-    processSize += event.getBody().length;
-    eventCounter++;
-  }
-
-  private void writeEventBatchToSerializer()
-      throws InterruptedException, WriteException {
-    try {
-      timedCall(new CallRunner1<Void>() {
-        @Override
-        public Void call() throws InterruptedException, StreamingException {
-          try {
-            for (Event event : batch) {
-              try {
-                serializer.write(txnBatch, event);
-              } catch (SerializationError err) {
-                LOG.info("Parse failed : {}  : {}", err.getMessage(), new String(event.getBody()));
-              }
-            }
-            return null;
-          } catch (IOException e) {
-            throw new StreamingIOFailure(e.getMessage(), e);
-          }
-        }
-      });
-      batch.clear();
-    } catch (StreamingException e) {
-      throw new WriteException(endPoint, txnBatch.getCurrentTxnId(), e);
-    } catch (TimeoutException e) {
-      throw new WriteException(endPoint, txnBatch.getCurrentTxnId(), e);
-    }
-  }
-
-  /**
-   * Commits the current Txn.
-   * If 'rollToNext' is true, will switch to next Txn in batch or to a
-   *       new TxnBatch if current Txn batch is exhausted
-   */
-  public void flush(boolean rollToNext)
-      throws CommitException, TxnBatchException, TxnFailure, InterruptedException,
-      WriteException {
-    if (!batch.isEmpty()) {
-      writeEventBatchToSerializer();
-      batch.clear();
-    }
-
-    //0 Heart beat on TxnBatch
-    if (hearbeatNeeded) {
-      hearbeatNeeded = false;
-      heartBeat();
-    }
-    lastUsed = System.currentTimeMillis();
-
-    try {
-      //1 commit txn & close batch if needed
-      commitTxn();
-      if (txnBatch.remainingTransactions() == 0) {
-        closeTxnBatch();
-        txnBatch = null;
-        if (rollToNext) {
-          txnBatch = nextTxnBatch(recordWriter);
-        }
-      }
-
-      //2 roll to next Txn
-      if (rollToNext) {
-        LOG.debug("Switching to next Txn for {}", endPoint);
-        txnBatch.beginNextTransaction(); // does not block
-      }
-    } catch (StreamingException e) {
-      throw new TxnFailure(txnBatch, e);
-    }
-  }
-
-  /**
-   * Aborts the current Txn
-   * @throws InterruptedException
-   */
-  public void abort() throws InterruptedException {
-    batch.clear();
-    abortTxn();
-  }
-
-  /** Queues up a heartbeat request on the current and remaining txns using the
-   *  heartbeatThdPool and returns immediately
-   */
-  public void heartBeat() throws InterruptedException {
-    // 1) schedule the heartbeat on one thread in pool
-    try {
-      timedCall(new CallRunner1<Void>() {
-        @Override
-        public Void call() throws StreamingException {
-          LOG.info("Sending heartbeat on batch " + txnBatch);
-          txnBatch.heartbeat();
-          return null;
-        }
-      });
-    } catch (InterruptedException e) {
-      throw e;
-    } catch (Exception e) {
-      LOG.warn("Unable to send heartbeat on Txn Batch " + txnBatch, e);
-      // Suppressing exceptions as we don't care for errors on heartbeats
-    }
-  }
-
-  /**
-   * Close the Transaction Batch and connection
-   * @throws IOException
-   * @throws InterruptedException
-   */
-  public void close() throws InterruptedException {
-    batch.clear();
-    abortRemainingTxns();
-    closeTxnBatch();
-    closeConnection();
-    closed = true;
-  }
-
-
-  private void abortRemainingTxns() throws InterruptedException {
-    try {
-      if (!isClosed(txnBatch.getCurrentTransactionState())) {
-        abortCurrTxnHelper();
-      }
-
-      // recursively abort remaining txns
-      if (txnBatch.remainingTransactions() > 0) {
-        timedCall(
-            new CallRunner1<Void>() {
-              @Override
-              public Void call() throws StreamingException, InterruptedException {
-                txnBatch.beginNextTransaction();
-                return null;
-              }
-            });
-        abortRemainingTxns();
-      }
-    } catch (StreamingException e) {
-      LOG.warn("Error when aborting remaining transactions in batch " + txnBatch, e);
-      return;
-    } catch (TimeoutException e) {
-      LOG.warn("Timed out when aborting remaining transactions in batch " + txnBatch, e);
-      return;
-    }
-  }
-
-  private void abortCurrTxnHelper() throws TimeoutException, InterruptedException {
-    try {
-      timedCall(
-          new CallRunner1<Void>() {
-            @Override
-            public Void call() throws StreamingException, InterruptedException {
-              txnBatch.abort();
-              LOG.info("Aborted txn " + txnBatch.getCurrentTxnId());
-              return null;
-            }
-          }
-      );
-    } catch (StreamingException e) {
-      LOG.warn("Unable to abort transaction " + txnBatch.getCurrentTxnId(), e);
-      // continue to attempt to abort other txns in the batch
-    }
-  }
-
-  private boolean isClosed(TransactionBatch.TxnState txnState) {
-    if (txnState == TransactionBatch.TxnState.COMMITTED) {
-      return true;
-    }
-    if (txnState == TransactionBatch.TxnState.ABORTED) {
-      return true;
-    }
-    return false;
-  }
-
-  public void closeConnection() throws InterruptedException {
-    LOG.info("Closing connection to EndPoint : {}", endPoint);
-    try {
-      timedCall(new CallRunner1<Void>() {
-        @Override
-        public Void call() {
-          connection.close(); // could block
-          return null;
-        }
-      });
-      sinkCounter.incrementConnectionClosedCount();
-    } catch (Exception e) {
-      LOG.warn("Error closing connection to EndPoint : " + endPoint, e);
-      // Suppressing exceptions as we don't care for errors on connection close
-    }
-  }
-
-  private void commitTxn() throws CommitException, InterruptedException {
-    if (LOG.isInfoEnabled()) {
-      LOG.info("Committing Txn " + txnBatch.getCurrentTxnId() + " on EndPoint: " + endPoint);
-    }
-    try {
-      timedCall(new CallRunner1<Void>() {
-        @Override
-        public Void call() throws StreamingException, InterruptedException {
-          txnBatch.commit(); // could block
-          return null;
-        }
-      });
-    } catch (Exception e) {
-      throw new CommitException(endPoint, txnBatch.getCurrentTxnId(), e);
-    }
-  }
-
-  private void abortTxn() throws InterruptedException {
-    LOG.info("Aborting Txn id {} on End Point {}", txnBatch.getCurrentTxnId(), endPoint);
-    try {
-      timedCall(new CallRunner1<Void>() {
-        @Override
-        public Void call() throws StreamingException, InterruptedException {
-          txnBatch.abort(); // could block
-          return null;
-        }
-      });
-    } catch (InterruptedException e) {
-      throw e;
-    } catch (TimeoutException e) {
-      LOG.warn("Timeout while aborting Txn " + txnBatch.getCurrentTxnId() +
-               " on EndPoint: " + endPoint, e);
-    } catch (Exception e) {
-      LOG.warn("Error aborting Txn " + txnBatch.getCurrentTxnId() + " on EndPoint: " + endPoint, e);
-      // Suppressing exceptions as we don't care for errors on abort
-    }
-  }
-
-  private StreamingConnection newConnection(final String proxyUser)
-      throws InterruptedException, ConnectException {
-    try {
-      return timedCall(new CallRunner1<StreamingConnection>() {
-        @Override
-        public StreamingConnection call() throws InterruptedException, StreamingException {
-          return endPoint.newConnection(autoCreatePartitions); // could block
-        }
-      });
-    } catch (Exception e) {
-      throw new ConnectException(endPoint, e);
-    }
-  }
-
-  private TransactionBatch nextTxnBatch(final RecordWriter recordWriter)
-      throws InterruptedException, TxnBatchException {
-    LOG.debug("Fetching new Txn Batch for {}", endPoint);
-    TransactionBatch batch = null;
-    try {
-      batch = timedCall(new CallRunner1<TransactionBatch>() {
-        @Override
-        public TransactionBatch call() throws InterruptedException, StreamingException {
-          return connection.fetchTransactionBatch(txnsPerBatch, recordWriter); // could block
-        }
-      });
-      LOG.info("Acquired Transaction batch {}", batch);
-    } catch (Exception e) {
-      throw new TxnBatchException(endPoint, e);
-    }
-    return batch;
-  }
-
-  private void closeTxnBatch() throws InterruptedException {
-    try {
-      LOG.info("Closing Txn Batch {}.", txnBatch);
-      timedCall(new CallRunner1<Void>() {
-        @Override
-        public Void call() throws InterruptedException, StreamingException {
-          txnBatch.close(); // could block
-          return null;
-        }
-      });
-    } catch (InterruptedException e) {
-      throw e;
-    } catch (Exception e) {
-      LOG.warn("Error closing Txn Batch " + txnBatch, e);
-      // Suppressing exceptions as we don't care for errors on batch close
-    }
-  }
-
-  private <T> T timedCall(final CallRunner1<T> callRunner)
-      throws TimeoutException, InterruptedException, StreamingException {
-    Future<T> future = callTimeoutPool.submit(new Callable<T>() {
-      @Override
-      public T call() throws StreamingException, InterruptedException, Failure {
-        return callRunner.call();
-      }
-    });
-
-    try {
-      if (callTimeout > 0) {
-        return future.get(callTimeout, TimeUnit.MILLISECONDS);
-      } else {
-        return future.get();
-      }
-    } catch (TimeoutException eT) {
-      future.cancel(true);
-      sinkCounter.incrementConnectionFailedCount();
-      throw eT;
-    } catch (ExecutionException e1) {
-      sinkCounter.incrementConnectionFailedCount();
-      Throwable cause = e1.getCause();
-      if (cause instanceof IOException) {
-        throw new StreamingException("I/O Failure", (IOException) cause);
-      } else if (cause instanceof StreamingException) {
-        throw (StreamingException) cause;
-      } else if (cause instanceof TimeoutException) {
-        throw new StreamingException("Operation Timed Out.", (TimeoutException) cause);
-      } else if (cause instanceof RuntimeException) {
-        throw (RuntimeException) cause;
-      } else if (cause instanceof InterruptedException) {
-        throw (InterruptedException) cause;
-      }
-      throw new StreamingException(e1.getMessage(), e1);
-    }
-  }
-
-  long getLastUsed() {
-    return lastUsed;
-  }
-
-  /**
-   * Simple interface whose <tt>call</tt> method is called by
-   * {#callWithTimeout} in a new thread inside a
-   * {@linkplain java.security.PrivilegedExceptionAction#run()} call.
-   * @param <T>
-   */
-  private interface CallRunner<T> {
-    T call() throws Exception;
-  }
-
-  private interface CallRunner1<T> {
-    T call() throws StreamingException, InterruptedException, Failure;
-  }
-
-  public static class Failure extends Exception {
-    public Failure(String msg, Throwable cause) {
-      super(msg, cause);
-    }
-  }
-
-  public static class WriteException extends Failure {
-    public WriteException(HiveEndPoint endPoint, Long currentTxnId, Throwable cause) {
-      super("Failed writing to : " + endPoint + ". TxnID : " + currentTxnId, cause);
-    }
-  }
-
-  public static class CommitException extends Failure {
-    public CommitException(HiveEndPoint endPoint, Long txnID, Throwable cause) {
-      super("Commit of Txn " + txnID + " failed on EndPoint: " + endPoint, cause);
-    }
-  }
-
-  public static class ConnectException extends Failure {
-    public ConnectException(HiveEndPoint ep, Throwable cause) {
-      super("Failed connecting to EndPoint " + ep, cause);
-    }
-  }
-
-  public static class TxnBatchException extends Failure {
-    public TxnBatchException(HiveEndPoint ep, Throwable cause) {
-      super("Failed acquiring Transaction Batch from EndPoint: " + ep, cause);
-    }
-  }
-
-  private class TxnFailure extends Failure {
-    public TxnFailure(TransactionBatch txnBatch, Throwable cause) {
-      super("Failed switching to next Txn in TxnBatch " + txnBatch, cause);
-    }
-  }
-}
diff --git a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveSink.java b/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveSink.java
deleted file mode 100644
index d07ef4bec..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveSink.java
+++ /dev/null
@@ -1,413 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.flume.sink.hive;
-
-import com.google.common.collect.Lists;
-import junit.framework.Assert;
-import org.apache.flume.Channel;
-import org.apache.flume.ChannelException;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.Transaction;
-import org.apache.flume.channel.BasicTransactionSemantics;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.conf.Configurables;
-import org.apache.flume.event.SimpleEvent;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.mockito.Mockito;
-import org.mockito.internal.util.reflection.Whitebox;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.List;
-import java.util.UUID;
-
-public class TestHiveSink {
-  // 1)  partitioned table
-  static final String dbName = "testing";
-  static final String tblName = "alerts";
-
-  public static final String PART1_NAME = "continent";
-  public static final String PART2_NAME = "country";
-  public static final String[] partNames = { PART1_NAME, PART2_NAME };
-
-  private static final String COL1 = "id";
-  private static final String COL2 = "msg";
-  final String[] colNames = {COL1,COL2};
-  private String[] colTypes = { "int", "string" };
-
-  private static final String PART1_VALUE = "Asia";
-  private static final String PART2_VALUE = "India";
-  private final ArrayList<String> partitionVals;
-
-  // 2) un-partitioned table
-  static final String dbName2 = "testing2";
-  static final String tblName2 = "alerts2";
-  final String[] colNames2 = {COL1,COL2};
-  private String[] colTypes2 = { "int", "string" };
-
-  HiveSink sink = new HiveSink();
-
-  private final HiveConf conf;
-
-  private final Driver driver;
-
-  final String metaStoreURI;
-
-  @Rule
-  public TemporaryFolder dbFolder = new TemporaryFolder();
-
-  private static final Logger LOG = LoggerFactory.getLogger(HiveSink.class);
-
-  public TestHiveSink() throws Exception {
-    partitionVals = new ArrayList<String>(2);
-    partitionVals.add(PART1_VALUE);
-    partitionVals.add(PART2_VALUE);
-
-    metaStoreURI = "null";
-
-    conf = new HiveConf(this.getClass());
-    TestUtil.setConfValues(conf);
-
-    // 1) prepare hive
-    TxnDbUtil.cleanDb(conf);
-    TxnDbUtil.prepDb(conf);
-
-    // 2) Setup Hive client
-    SessionState.start(new CliSessionState(conf));
-    driver = new Driver(conf);
-
-  }
-
-
-  @Before
-  public void setUp() throws Exception {
-    TestUtil.dropDB(conf, dbName);
-
-    sink = new HiveSink();
-    sink.setName("HiveSink-" + UUID.randomUUID().toString());
-
-    String dbLocation = dbFolder.newFolder(dbName).getCanonicalPath() + ".db";
-    dbLocation = dbLocation.replaceAll("\\\\","/"); // for windows paths
-    TestUtil.createDbAndTable(driver, dbName, tblName, partitionVals, colNames,
-            colTypes, partNames, dbLocation);
-  }
-
-  @After
-  public void tearDown() throws MetaException, HiveException {
-    TestUtil.dropDB(conf, dbName);
-  }
-
-  public void testSingleWriter(boolean partitioned, String dbName, String tblName,
-                               Channel pChannel) throws Exception {
-    int totalRecords = 4;
-    int batchSize = 2;
-    int batchCount = totalRecords / batchSize;
-
-    Context context = new Context();
-    context.put("hive.metastore", metaStoreURI);
-    context.put("hive.database",dbName);
-    context.put("hive.table",tblName);
-    if (partitioned) {
-      context.put("hive.partition", PART1_VALUE + "," + PART2_VALUE);
-    }
-    context.put("autoCreatePartitions","false");
-    context.put("batchSize","" + batchSize);
-    context.put("serializer", HiveDelimitedTextSerializer.ALIAS);
-    context.put("serializer.fieldnames", COL1 + ",," + COL2 + ",");
-    context.put("heartBeatInterval", "0");
-
-    Channel channel = startSink(sink, context, pChannel);
-
-    List<String> bodies = Lists.newArrayList();
-
-    // push the events in two batches
-    Transaction txn = channel.getTransaction();
-    txn.begin();
-    for (int j = 1; j <= totalRecords; j++) {
-      Event event = new SimpleEvent();
-      String body = j + ",blah,This is a log message,other stuff";
-      event.setBody(body.getBytes());
-      bodies.add(body);
-      channel.put(event);
-    }
-    // execute sink to process the events
-    txn.commit();
-    txn.close();
-
-
-    checkRecordCountInTable(0, dbName, tblName);
-    for (int i = 0; i < batchCount ; i++) {
-      sink.process();
-    }
-    checkRecordCountInTable(totalRecords, dbName, tblName);
-    sink.stop();
-    checkRecordCountInTable(totalRecords, dbName, tblName);
-  }
-
-  @Test
-  public void testSingleWriterSimplePartitionedTable() throws Exception {
-    testSingleWriter(true, dbName, tblName, null);
-  }
-
-  @Test
-  public void testSingleWriterSimpleUnPartitionedTable()
-          throws Exception {
-    TestUtil.dropDB(conf, dbName2);
-    String dbLocation = dbFolder.newFolder(dbName2).getCanonicalPath() + ".db";
-    dbLocation = dbLocation.replaceAll("\\\\","/"); // for windows paths
-    TestUtil.createDbAndTable(driver, dbName2, tblName2, null, colNames2, colTypes2,
-                              null, dbLocation);
-
-    try {
-      testSingleWriter(false, dbName2, tblName2, null);
-    } finally {
-      TestUtil.dropDB(conf, dbName2);
-    }
-  }
-
-  @Test
-  public void testSingleWriterUseHeaders()
-          throws Exception {
-    String[] colNames = {COL1, COL2};
-    String PART1_NAME = "country";
-    String PART2_NAME = "hour";
-    String[] partNames = {PART1_NAME, PART2_NAME};
-    List<String> partitionVals = null;
-    String PART1_VALUE = "%{" + PART1_NAME + "}";
-    String PART2_VALUE = "%y-%m-%d-%k";
-    partitionVals = new ArrayList<String>(2);
-    partitionVals.add(PART1_VALUE);
-    partitionVals.add(PART2_VALUE);
-
-    String tblName = "hourlydata";
-    TestUtil.dropDB(conf, dbName2);
-    String dbLocation = dbFolder.newFolder(dbName2).getCanonicalPath() + ".db";
-    dbLocation = dbLocation.replaceAll("\\\\","/"); // for windows paths
-    TestUtil.createDbAndTable(driver, dbName2, tblName, partitionVals, colNames,
-            colTypes, partNames, dbLocation);
-
-    int totalRecords = 4;
-    int batchSize = 2;
-    int batchCount = totalRecords / batchSize;
-
-    Context context = new Context();
-    context.put("hive.metastore",metaStoreURI);
-    context.put("hive.database",dbName2);
-    context.put("hive.table",tblName);
-    context.put("hive.partition", PART1_VALUE + "," + PART2_VALUE);
-    context.put("autoCreatePartitions","true");
-    context.put("useLocalTimeStamp", "false");
-    context.put("batchSize","" + batchSize);
-    context.put("serializer", HiveDelimitedTextSerializer.ALIAS);
-    context.put("serializer.fieldnames", COL1 + ",," + COL2 + ",");
-    context.put("heartBeatInterval", "0");
-
-    Channel channel = startSink(sink, context);
-
-    Calendar eventDate = Calendar.getInstance();
-    List<String> bodies = Lists.newArrayList();
-
-    // push events in two batches - two per batch. each batch is diff hour
-    Transaction txn = channel.getTransaction();
-    txn.begin();
-    for (int j = 1; j <= totalRecords; j++) {
-      Event event = new SimpleEvent();
-      String body = j + ",blah,This is a log message,other stuff";
-      event.setBody(body.getBytes());
-      eventDate.clear();
-      eventDate.set(2014, 03, 03, j % batchCount, 1); // yy mm dd hh mm
-      event.getHeaders().put( "timestamp",
-              String.valueOf(eventDate.getTimeInMillis()) );
-      event.getHeaders().put( PART1_NAME, "Asia" );
-      bodies.add(body);
-      channel.put(event);
-    }
-    // execute sink to process the events
-    txn.commit();
-    txn.close();
-
-    checkRecordCountInTable(0, dbName2, tblName);
-    for (int i = 0; i < batchCount ; i++) {
-      sink.process();
-    }
-    checkRecordCountInTable(totalRecords, dbName2, tblName);
-    sink.stop();
-
-    // verify counters
-    SinkCounter counter = sink.getCounter();
-    Assert.assertEquals(2, counter.getConnectionCreatedCount());
-    Assert.assertEquals(2, counter.getConnectionClosedCount());
-    Assert.assertEquals(2, counter.getBatchCompleteCount());
-    Assert.assertEquals(0, counter.getBatchEmptyCount());
-    Assert.assertEquals(0, counter.getConnectionFailedCount() );
-    Assert.assertEquals(4, counter.getEventDrainAttemptCount());
-    Assert.assertEquals(4, counter.getEventDrainSuccessCount() );
-
-  }
-
-  @Test
-  public void testHeartBeat()
-          throws EventDeliveryException, IOException {
-    int batchSize = 2;
-    int batchCount = 3;
-    int totalRecords = batchCount * batchSize;
-    Context context = new Context();
-    context.put("hive.metastore", metaStoreURI);
-    context.put("hive.database", dbName);
-    context.put("hive.table", tblName);
-    context.put("hive.partition", PART1_VALUE + "," + PART2_VALUE);
-    context.put("autoCreatePartitions","true");
-    context.put("batchSize","" + batchSize);
-    context.put("serializer", HiveDelimitedTextSerializer.ALIAS);
-    context.put("serializer.fieldnames", COL1 + ",," + COL2 + ",");
-    context.put("hive.txnsPerBatchAsk", "20");
-    context.put("heartBeatInterval", "3"); // heartbeat in seconds
-
-    Channel channel = startSink(sink, context);
-
-    List<String> bodies = Lists.newArrayList();
-
-    // push the events in two batches
-    for (int i = 0; i < batchCount; i++) {
-      Transaction txn = channel.getTransaction();
-      txn.begin();
-      for (int j = 1; j <= batchSize; j++) {
-        Event event = new SimpleEvent();
-        String body = i * j + ",blah,This is a log message,other stuff";
-        event.setBody(body.getBytes());
-        bodies.add(body);
-        channel.put(event);
-      }
-      // execute sink to process the events
-      txn.commit();
-      txn.close();
-
-      sink.process();
-      sleep(3000); // allow heartbeat to happen
-    }
-
-    sink.stop();
-    checkRecordCountInTable(totalRecords, dbName, tblName);
-  }
-
-  @Test
-  public void testJsonSerializer() throws Exception {
-    int batchSize = 2;
-    int batchCount = 2;
-    int totalRecords = batchCount * batchSize;
-    Context context = new Context();
-    context.put("hive.metastore",metaStoreURI);
-    context.put("hive.database",dbName);
-    context.put("hive.table",tblName);
-    context.put("hive.partition", PART1_VALUE + "," + PART2_VALUE);
-    context.put("autoCreatePartitions","true");
-    context.put("batchSize","" + batchSize);
-    context.put("serializer", HiveJsonSerializer.ALIAS);
-    context.put("serializer.fieldnames", COL1 + ",," + COL2 + ",");
-    context.put("heartBeatInterval", "0");
-
-    Channel channel = startSink(sink, context);
-
-    List<String> bodies = Lists.newArrayList();
-
-    // push the events in two batches
-    for (int i = 0; i < batchCount; i++) {
-      Transaction txn = channel.getTransaction();
-      txn.begin();
-      for (int j = 1; j <= batchSize; j++) {
-        Event event = new SimpleEvent();
-        String body = "{\"id\" : 1, \"msg\" : \"using json serializer\"}";
-        event.setBody(body.getBytes());
-        bodies.add(body);
-        channel.put(event);
-      }
-      // execute sink to process the events
-      txn.commit();
-      txn.close();
-
-      sink.process();
-    }
-    checkRecordCountInTable(totalRecords, dbName, tblName);
-    sink.stop();
-    checkRecordCountInTable(totalRecords, dbName, tblName);
-  }
-
-  @Test
-  public void testErrorCounter() throws Exception {
-    Channel channel = Mockito.mock(Channel.class);
-    Mockito.when(channel.take()).thenThrow(new ChannelException("dummy"));
-    Transaction transaction = Mockito.mock(BasicTransactionSemantics.class);
-    Mockito.when(channel.getTransaction()).thenReturn(transaction);
-
-    try {
-      testSingleWriter(true, dbName, tblName, channel);
-    } catch (EventDeliveryException e) {
-      //Expected exception
-    }
-
-    SinkCounter sinkCounter = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
-    Assert.assertEquals(1, sinkCounter.getChannelReadFail());
-  }
-
-  private void sleep(int n) {
-    try {
-      Thread.sleep(n);
-    } catch (InterruptedException e) {
-    }
-  }
-
-  private static Channel startSink(HiveSink sink, Context context) {
-    return startSink(sink, context, null);
-  }
-
-  private static Channel startSink(HiveSink sink, Context context, Channel pChannel) {
-    Configurables.configure(sink, context);
-
-    Channel channel = pChannel == null ? new MemoryChannel() : pChannel;
-    Configurables.configure(channel, context);
-    sink.setChannel(channel);
-    sink.start();
-    return channel;
-  }
-
-  private void checkRecordCountInTable(int expectedCount, String db, String tbl)
-          throws IOException {
-    int count = TestUtil.listRecordsInTable(driver, db, tbl).size();
-    Assert.assertEquals(expectedCount, count);
-  }
-}
diff --git a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveWriter.java b/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveWriter.java
deleted file mode 100644
index 1e8ce1fec..000000000
--- a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveWriter.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flume.sink.hive;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import junit.framework.Assert;
-import org.apache.flume.Context;
-import org.apache.flume.event.SimpleEvent;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class TestHiveWriter {
-  static final String dbName = "testing";
-  static final String tblName = "alerts";
-
-  public static final String PART1_NAME = "continent";
-  public static final String PART2_NAME = "country";
-  public static final String[] partNames = { PART1_NAME, PART2_NAME };
-
-  private static final String COL1 = "id";
-  private static final String COL2 = "msg";
-  final String[] colNames = {COL1,COL2};
-  private String[] colTypes = { "int", "string" };
-
-  private static final String PART1_VALUE = "Asia";
-  private static final String PART2_VALUE = "India";
-  private final ArrayList<String> partVals;
-
-  private final String metaStoreURI;
-
-  private HiveDelimitedTextSerializer serializer;
-
-  private final HiveConf conf;
-
-  private ExecutorService callTimeoutPool;
-  int timeout = 10000; // msec
-
-  @Rule
-  public TemporaryFolder dbFolder = new TemporaryFolder();
-
-  private final Driver driver;
-
-  public TestHiveWriter() throws Exception {
-    partVals = new ArrayList<String>(2);
-    partVals.add(PART1_VALUE);
-    partVals.add(PART2_VALUE);
-
-    metaStoreURI = null;
-
-    int callTimeoutPoolSize = 1;
-    callTimeoutPool = Executors.newFixedThreadPool(callTimeoutPoolSize,
-            new ThreadFactoryBuilder().setNameFormat("hiveWriterTest").build());
-
-    // 1) Start metastore
-    conf = new HiveConf(this.getClass());
-    TestUtil.setConfValues(conf);
-    if (metaStoreURI != null) {
-      conf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreURI);
-    }
-
-    // 2) Setup Hive client
-    SessionState.start(new CliSessionState(conf));
-    driver = new Driver(conf);
-
-  }
-
-  @Before
-  public void setUp() throws Exception {
-    // 1) prepare hive
-    TxnDbUtil.cleanDb(conf);
-    TxnDbUtil.prepDb(conf);
-
-    // 1) Setup tables
-    TestUtil.dropDB(conf, dbName);
-    String dbLocation = dbFolder.newFolder(dbName).getCanonicalPath() + ".db";
-    dbLocation = dbLocation.replaceAll("\\\\","/"); // for windows paths
-    TestUtil.createDbAndTable(driver, dbName, tblName, partVals, colNames, colTypes, partNames,
-                              dbLocation);
-
-    // 2) Setup serializer
-    Context ctx = new Context();
-    ctx.put("serializer.fieldnames", COL1 + ",," + COL2 + ",");
-    serializer = new HiveDelimitedTextSerializer();
-    serializer.configure(ctx);
-  }
-
-  @Test
-  public void testInstantiate() throws Exception {
-    HiveEndPoint endPoint = new HiveEndPoint(metaStoreURI, dbName, tblName, partVals);
-    SinkCounter sinkCounter = new SinkCounter(this.getClass().getName());
-    HiveWriter writer = new HiveWriter(endPoint, 10, true, timeout, callTimeoutPool, "flumetest",
-                                       serializer, sinkCounter);
-
-    writer.close();
-  }
-
-  @Test
-  public void testWriteBasic() throws Exception {
-    HiveEndPoint endPoint = new HiveEndPoint(metaStoreURI, dbName, tblName, partVals);
-    SinkCounter sinkCounter = new SinkCounter(this.getClass().getName());
-    HiveWriter writer = new HiveWriter(endPoint, 10, true, timeout, callTimeoutPool, "flumetest",
-                                       serializer, sinkCounter);
-
-    writeEvents(writer,3);
-    writer.flush(false);
-    writer.close();
... 6117 lines suppressed ...