You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flume.apache.org by rg...@apache.org on 2023/04/07 04:53:20 UTC
[flume-hadoop] branch main created (now 8fb2d44)
This is an automated email from the ASF dual-hosted git repository.
rgoers pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/flume-hadoop.git
at 8fb2d44 FLUME-3467 - Move Hadoop based components to a separate repo
This branch includes the following new commits:
new 8fb2d44 FLUME-3467 - Move Hadoop based components to a separate repo
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[flume-hadoop] 01/01: FLUME-3467 - Move Hadoop based components to a separate repo
Posted by rg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rgoers pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flume-hadoop.git
commit 8fb2d44793f7b39713c94f88df84f66818726e92
Author: Ralph Goers <rg...@apache.org>
AuthorDate: Thu Apr 6 21:53:11 2023 -0700
FLUME-3467 - Move Hadoop based components to a separate repo
---
.asf.yaml | 40 +
CHANGELOG | 10 +
LICENSE.txt | 202 +++
NOTICE.txt | 5 +
README.md | 58 +
RELEASE-NOTES.txt | 26 +
checkstyle-header.txt | 16 +
findbugs-exclude-filter.xml | 31 +
.../pom.xml | 59 +
.../HadoopCredentialStoreConfigFilter.java | 77 +
.../TestHadoopCredentialStoreConfigFilter.java | 183 +++
.../src/test/resources/test-password.txt | 1 +
.../src/test/resources/test-password2.txt | 1 +
flume-hadoop-dist/pom.xml | 159 ++
flume-hadoop-dist/src/assembly/bin.xml | 54 +
flume-hadoop-dist/src/assembly/src.xml | 45 +
flume-hbase2-sink/pom.xml | 123 ++
.../org/apache/flume/sink/hbase2/BatchAware.java | 28 +
.../flume/sink/hbase2/HBase2EventSerializer.java | 61 +
.../org/apache/flume/sink/hbase2/HBase2Sink.java | 555 +++++++
.../hbase2/HBase2SinkConfigurationConstants.java | 77 +
.../sink/hbase2/RegexHBase2EventSerializer.java | 214 +++
.../sink/hbase2/SimpleHBase2EventSerializer.java | 151 ++
.../flume/sink/hbase2/SimpleRowKeyGenerator.java | 46 +
.../sink/hbase2/IncrementHBase2Serializer.java | 83 +
.../hbase2/MockSimpleHBase2EventSerializer.java | 38 +
.../apache/flume/sink/hbase2/TestHBase2Sink.java | 786 +++++++++
.../flume/sink/hbase2/TestHBase2SinkCreation.java | 48 +
.../hbase2/TestRegexHBase2EventSerializer.java | 238 +++
flume-hdfs-sink/pom.xml | 133 ++
.../apache/flume/sink/hdfs/AbstractHDFSWriter.java | 281 ++++
.../flume/sink/hdfs/AvroEventSerializer.java | 211 +++
.../flume/sink/hdfs/BucketClosedException.java | 30 +
.../org/apache/flume/sink/hdfs/BucketWriter.java | 776 +++++++++
.../flume/sink/hdfs/HDFSCompressedDataStream.java | 162 ++
.../org/apache/flume/sink/hdfs/HDFSDataStream.java | 140 ++
.../org/apache/flume/sink/hdfs/HDFSEventSink.java | 577 +++++++
.../apache/flume/sink/hdfs/HDFSSequenceFile.java | 122 ++
.../apache/flume/sink/hdfs/HDFSTextSerializer.java | 79 +
.../flume/sink/hdfs/HDFSWritableSerializer.java | 77 +
.../org/apache/flume/sink/hdfs/HDFSWriter.java | 47 +
.../apache/flume/sink/hdfs/HDFSWriterFactory.java | 43 +
.../org/apache/flume/sink/hdfs/KerberosUser.java | 72 +
.../flume/sink/hdfs/SequenceFileSerializer.java | 68 +
.../sink/hdfs/SequenceFileSerializerFactory.java | 90 ++
.../sink/hdfs/SequenceFileSerializerType.java | 37 +
.../apache/flume/sink/hdfs/HDFSBadDataStream.java | 45 +
.../apache/flume/sink/hdfs/HDFSTestSeqWriter.java | 77 +
.../flume/sink/hdfs/HDFSTestWriterFactory.java | 42 +
.../org/apache/flume/sink/hdfs/MockDataStream.java | 39 +
.../org/apache/flume/sink/hdfs/MockFileSystem.java | 139 ++
.../flume/sink/hdfs/MockFsDataOutputStream.java | 49 +
.../org/apache/flume/sink/hdfs/MockHDFSWriter.java | 111 ++
.../apache/flume/sink/hdfs/MyCustomSerializer.java | 58 +
.../flume/sink/hdfs/TestAvroEventSerializer.java | 183 +++
.../apache/flume/sink/hdfs/TestBucketWriter.java | 719 +++++++++
.../sink/hdfs/TestHDFSCompressedDataStream.java | 143 ++
.../apache/flume/sink/hdfs/TestHDFSEventSink.java | 1687 ++++++++++++++++++++
.../flume/sink/hdfs/TestHDFSEventSinkDeadlock.java | 127 ++
.../sink/hdfs/TestHDFSEventSinkOnMiniCluster.java | 607 +++++++
.../hdfs/TestSequenceFileSerializerFactory.java | 58 +
.../flume/sink/hdfs/TestUseRawLocalFileSystem.java | 95 ++
flume-hdfs-sink/src/test/resources/log4j2.xml | 33 +
flume-hive-sink/derby.log | 13 +
.../metastore_db/README_DO_NOT_TOUCH_FILES.txt | 9 +
flume-hive-sink/metastore_db/db.lck | Bin 0 -> 38 bytes
flume-hive-sink/metastore_db/dbex.lck | Bin 0 -> 4 bytes
.../metastore_db/log/README_DO_NOT_TOUCH_FILES.txt | 8 +
flume-hive-sink/metastore_db/log/log.ctrl | Bin 0 -> 48 bytes
flume-hive-sink/metastore_db/log/log32.dat | Bin 0 -> 1050157 bytes
flume-hive-sink/metastore_db/log/log33.dat | Bin 0 -> 1048628 bytes
flume-hive-sink/metastore_db/log/log34.dat | Bin 0 -> 1081748 bytes
flume-hive-sink/metastore_db/log/log35.dat | Bin 0 -> 1048576 bytes
flume-hive-sink/metastore_db/log/logmirror.ctrl | Bin 0 -> 48 bytes
.../seg0/README_DO_NOT_TOUCH_FILES.txt | 8 +
flume-hive-sink/metastore_db/seg0/c10.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c101.dat | Bin 0 -> 16384 bytes
flume-hive-sink/metastore_db/seg0/c111.dat | Bin 0 -> 16384 bytes
flume-hive-sink/metastore_db/seg0/c121.dat | Bin 0 -> 16384 bytes
flume-hive-sink/metastore_db/seg0/c130.dat | Bin 0 -> 16384 bytes
flume-hive-sink/metastore_db/seg0/c141.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c150.dat | Bin 0 -> 24576 bytes
flume-hive-sink/metastore_db/seg0/c161.dat | Bin 0 -> 20480 bytes
flume-hive-sink/metastore_db/seg0/c171.dat | Bin 0 -> 16384 bytes
flume-hive-sink/metastore_db/seg0/c180.dat | Bin 0 -> 94208 bytes
flume-hive-sink/metastore_db/seg0/c191.dat | Bin 0 -> 16384 bytes
flume-hive-sink/metastore_db/seg0/c1a1.dat | Bin 0 -> 16384 bytes
flume-hive-sink/metastore_db/seg0/c1b1.dat | Bin 0 -> 16384 bytes
flume-hive-sink/metastore_db/seg0/c1c0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c1d1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c1e0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c1f1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c20.dat | Bin 0 -> 102400 bytes
flume-hive-sink/metastore_db/seg0/c200.dat | Bin 0 -> 16384 bytes
flume-hive-sink/metastore_db/seg0/c211.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c221.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c230.dat | Bin 0 -> 315392 bytes
flume-hive-sink/metastore_db/seg0/c241.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c251.dat | Bin 0 -> 16384 bytes
flume-hive-sink/metastore_db/seg0/c260.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c271.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c281.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c290.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c2a1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c2b1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c2c1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c2d0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c2e1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c2f0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c300.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c31.dat | Bin 0 -> 32768 bytes
flume-hive-sink/metastore_db/seg0/c311.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c321.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c331.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c340.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c351.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c361.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c371.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c380.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c391.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c3a1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c3b1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c3c0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c3d1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c3e1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c3f1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c400.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c41.dat | Bin 0 -> 69632 bytes
flume-hive-sink/metastore_db/seg0/c411.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c421.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c430.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c441.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c451.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c461.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c470.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c481.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c51.dat | Bin 0 -> 32768 bytes
flume-hive-sink/metastore_db/seg0/c60.dat | Bin 0 -> 20480 bytes
flume-hive-sink/metastore_db/seg0/c610.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c621.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c630.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c640.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c650.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/c660.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c671.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c680.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/c691.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c6a1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c6b0.dat | Bin 0 -> 557056 bytes
flume-hive-sink/metastore_db/seg0/c6c1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c6d0.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/c6e1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c6f1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c701.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c71.dat | Bin 0 -> 16384 bytes
flume-hive-sink/metastore_db/seg0/c710.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c721.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c731.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c740.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c751.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c761.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/c770.dat | Bin 0 -> 16384 bytes
flume-hive-sink/metastore_db/seg0/c781.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c790.dat | Bin 0 -> 131072 bytes
flume-hive-sink/metastore_db/seg0/c7a1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c7b0.dat | Bin 0 -> 98304 bytes
flume-hive-sink/metastore_db/seg0/c7c1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c7d0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c7e1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c7f0.dat | Bin 0 -> 98304 bytes
flume-hive-sink/metastore_db/seg0/c801.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c81.dat | Bin 0 -> 16384 bytes
flume-hive-sink/metastore_db/seg0/c810.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c821.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c830.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c841.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c850.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c861.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c870.dat | Bin 0 -> 12288 bytes
flume-hive-sink/metastore_db/seg0/c881.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c890.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/c8a1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c8b0.dat | Bin 0 -> 98304 bytes
flume-hive-sink/metastore_db/seg0/c8c1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c8d0.dat | Bin 0 -> 98304 bytes
flume-hive-sink/metastore_db/seg0/c8e1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c8f0.dat | Bin 0 -> 98304 bytes
flume-hive-sink/metastore_db/seg0/c90.dat | Bin 0 -> 102400 bytes
flume-hive-sink/metastore_db/seg0/c901.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/c910.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c921.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c930.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/c941.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c950.dat | Bin 0 -> 98304 bytes
flume-hive-sink/metastore_db/seg0/c961.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c971.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c981.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c991.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c9a1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c9b1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c9c1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c9d1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c9e1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/c9f1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ca01.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ca1.dat | Bin 0 -> 114688 bytes
flume-hive-sink/metastore_db/seg0/ca11.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ca21.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ca31.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ca41.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ca51.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ca61.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ca71.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ca81.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ca90.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/caa1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cab0.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/cac1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cad1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cae1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/caf1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb00.dat | Bin 0 -> 12288 bytes
flume-hive-sink/metastore_db/seg0/cb1.dat | Bin 0 -> 20480 bytes
flume-hive-sink/metastore_db/seg0/cb11.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb21.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/cb31.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb40.dat | Bin 0 -> 12288 bytes
flume-hive-sink/metastore_db/seg0/cb51.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb60.dat | Bin 0 -> 98304 bytes
flume-hive-sink/metastore_db/seg0/cb71.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb80.dat | Bin 0 -> 12288 bytes
flume-hive-sink/metastore_db/seg0/cb8a0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb8b1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb8c0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb8d1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb8e0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb8f0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb900.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb91.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb910.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb920.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb931.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb940.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb951.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb961.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb970.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb980.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/cb991.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb9a0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb9b0.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/cb9c1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb9d0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb9e1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cb9f0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cba00.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cba1.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/cba11.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cbb1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cbc1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cbd1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cbe1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cc0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cd1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cd80.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cd91.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cda1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cdb1.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/cdc0.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cdd1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cde1.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/cdf1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ce00.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/ce1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ce11.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ce21.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ce30.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ce41.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ce51.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ce61.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ce71.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ce81.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ce90.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cea1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ceb1.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/cec1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/ced0.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/cee1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cef1.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cf0.dat | Bin 0 -> 24576 bytes
flume-hive-sink/metastore_db/seg0/cf00.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cf11.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cf21.dat | Bin 0 -> 8192 bytes
flume-hive-sink/metastore_db/seg0/cf31.dat | Bin 0 -> 65536 bytes
flume-hive-sink/metastore_db/seg0/daf50.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/daf61.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/daf70.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/daf81.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/daf90.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/dafa0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/dafb0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/dafc0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/dafd0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/dafe1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/daff0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db001.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db011.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db020.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db030.dat | Bin 0 -> 32768 bytes
flume-hive-sink/metastore_db/seg0/db041.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db050.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db060.dat | Bin 0 -> 32768 bytes
flume-hive-sink/metastore_db/seg0/db071.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db080.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db091.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db0a0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db0b0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db0c1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db0d0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db0e1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db0f0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db101.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db110.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db120.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db130.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db140.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db150.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db161.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db170.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db181.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db191.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db1a0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db1b0.dat | Bin 0 -> 32768 bytes
flume-hive-sink/metastore_db/seg0/db1c1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db1d0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db1e0.dat | Bin 0 -> 32768 bytes
flume-hive-sink/metastore_db/seg0/db1f1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db200.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db211.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db220.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db230.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db241.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db250.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db260.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db271.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db280.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db291.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db2a0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db2b0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db2c0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db2d0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db2e0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db2f1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db300.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db311.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db321.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db330.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db340.dat | Bin 0 -> 32768 bytes
flume-hive-sink/metastore_db/seg0/db351.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db360.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db370.dat | Bin 0 -> 32768 bytes
flume-hive-sink/metastore_db/seg0/db381.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db390.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db3a1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db3b0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db3c0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db3d1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db3e0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db3f0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db401.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db410.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db421.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db430.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db440.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db450.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db460.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db470.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db481.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db490.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db4a1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db4b1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db4c0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db4d0.dat | Bin 0 -> 32768 bytes
flume-hive-sink/metastore_db/seg0/db4e1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db4f0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db500.dat | Bin 0 -> 32768 bytes
flume-hive-sink/metastore_db/seg0/db511.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db520.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db531.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db540.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db550.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db561.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db570.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db580.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db591.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db5a0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db5b1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db5c0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db5d0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db5e0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db5f0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db600.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db611.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db620.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db631.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db641.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db650.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db660.dat | Bin 0 -> 32768 bytes
flume-hive-sink/metastore_db/seg0/db671.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db680.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db690.dat | Bin 0 -> 32768 bytes
flume-hive-sink/metastore_db/seg0/db6a1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db6b0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db6c1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db6d0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db6e0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db6f1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db700.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db710.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db721.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db730.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db741.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db750.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db760.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db770.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db780.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db790.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db7a1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db7b0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db7c1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db7d1.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db7e0.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db7f0.dat | Bin 0 -> 32768 bytes
flume-hive-sink/metastore_db/seg0/db801.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db810.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db820.dat | Bin 0 -> 32768 bytes
flume-hive-sink/metastore_db/seg0/db831.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db840.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db851.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db860.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db870.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db881.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/db890.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/seg0/dba20.dat | Bin 0 -> 4096 bytes
flume-hive-sink/metastore_db/service.properties | 23 +
flume-hive-sink/pom.xml | 138 ++
.../java/org/apache/flume/sink/hive/Config.java | 41 +
.../sink/hive/HiveDelimitedTextSerializer.java | 115 ++
.../flume/sink/hive/HiveEventSerializer.java | 41 +
.../apache/flume/sink/hive/HiveJsonSerializer.java | 62 +
.../java/org/apache/flume/sink/hive/HiveSink.java | 528 ++++++
.../org/apache/flume/sink/hive/HiveWriter.java | 513 ++++++
.../org/apache/flume/sink/hive/TestHiveSink.java | 413 +++++
.../org/apache/flume/sink/hive/TestHiveWriter.java | 350 ++++
.../java/org/apache/flume/sink/hive/TestUtil.java | 235 +++
flume-hive-sink/src/test/resources/log4j2.xml | 33 +
flume-kudu-sink/pom.xml | 195 +++
.../sink/kudu/AvroKuduOperationsProducer.java | 282 ++++
.../flume/sink/kudu/KuduOperationsProducer.java | 54 +
.../java/org/apache/flume/sink/kudu/KuduSink.java | 315 ++++
.../sink/kudu/KuduSinkConfigurationConstants.java | 77 +
.../sink/kudu/RegexpKuduOperationsProducer.java | 398 +++++
.../kudu/SimpleKeyedKuduOperationsProducer.java | 136 ++
.../sink/kudu/SimpleKuduOperationsProducer.java | 92 ++
.../apache/flume/sink/kudu/KuduSinkTestUtil.java | 99 ++
.../sink/kudu/TestAvroKuduOperationsProducer.java | 183 +++
.../sink/kudu/TestKeyedKuduOperationsProducer.java | 189 +++
.../org/apache/flume/sink/kudu/TestKuduSink.java | 213 +++
.../kudu/TestRegexpKuduOperationsProducer.java | 184 +++
...TestRegexpKuduOperationsProducerParseError.java | 291 ++++
.../apache/flume/sink/kudu/TestSecureKuduSink.java | 126 ++
flume-kudu-sink/src/test/resources/log4j2.xml | 32 +
.../resources/testAvroKuduOperationsProducer.avsc | 11 +
pom.xml | 671 ++++++++
473 files changed, 16650 insertions(+)
diff --git a/.asf.yaml b/.asf.yaml
new file mode 100644
index 0000000..e0de80a
--- /dev/null
+++ b/.asf.yaml
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# `.asf.yaml` is a branch-specific YAML configuration file for Git repositories to control features such as notifications, GitHub settings, etc.
+# See its documentation for details: https://cwiki.apache.org/confluence/display/INFRA/Git+-+.asf.yaml+features
+
+notifications:
+ # GitHub already provides notifications for PRs and issues.
+ # Please don't duplicate that noise here!
+ commits: commits@flume.apache.org
+ jira_options: link label
+github:
+ description: "Apache Flume Hadoop provides various Flume components for the Hadoop ecosystem"
+ homepage: https://flume.apache.org/
+ features:
+ issues: true
+ del_branch_on_merge: true
+ autolink_jira:
+ - FLUME
+ labels:
+ - apache
+ - api
+ - java
+ - jvm
+ - library
+ - flume
+ protected_branches:
+ main: {}
diff --git a/CHANGELOG b/CHANGELOG
new file mode 100644
index 0000000..1cdf332
--- /dev/null
+++ b/CHANGELOG
@@ -0,0 +1,10 @@
+Release Notes - Flume Hadoop - Version 2.0.0
+
+** Bug
+ *
+
+** Improvement
+ * [FLUME-3467] - Move the Hadoop components to their own repo.
+
+
+
diff --git a/LICENSE.txt b/LICENSE.txt
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/LICENSE.txt
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/NOTICE.txt b/NOTICE.txt
new file mode 100644
index 0000000..a2c2ff9
--- /dev/null
+++ b/NOTICE.txt
@@ -0,0 +1,5 @@
+Apache Flume Spring Boot
+Copyright 2022-2023 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..e051bd2
--- /dev/null
+++ b/README.md
@@ -0,0 +1,58 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Welcome to Apache Flume Hadoop!
+
+Apache Flume is a distributed, reliable, and available service for efficiently
+collecting, aggregating, and moving large amounts of event data. It has a simple
+and flexible architecture based on streaming data flows. It is robust and fault
+tolerant with tunable reliability mechanisms and many failover and recovery
+mechanisms. The system is centrally managed and allows for intelligent dynamic
+management. It uses a simple extensible data model that allows for online
+analytic application.
+
+The Apache Flume Hadoop module provides Flume components that leverage Hadoop technologies.
+
+Apache Flume Hadoop is open-sourced under the Apache Software Foundation License v2.0.
+
+## Documentation
+
+Documentation is included in the binary distribution under the docs directory.
+In source form, it can be found in the flume-ng-doc directory.
+
+The Flume 1.x guide and FAQ are available here:
+
+* https://cwiki.apache.org/FLUME
+* https://cwiki.apache.org/confluence/display/FLUME/Getting+Started
+
+## Contact us!
+
+* Mailing lists: https://cwiki.apache.org/confluence/display/FLUME/Mailing+Lists
+* Slack channel #flume on https://the-asf.slack.com/
+
+Bug and Issue tracker.
+
+* https://github.com/apache/flume-Hadoop/issues
+
+## Compiling Flume Hadoop
+
+Compiling Flume Hadoop requires the following tools:
+
+* Oracle Java JDK 8
+* Apache Maven 3.x
diff --git a/RELEASE-NOTES.txt b/RELEASE-NOTES.txt
new file mode 100644
index 0000000..b2efb10
--- /dev/null
+++ b/RELEASE-NOTES.txt
@@ -0,0 +1,26 @@
+Apache Flume Hadoop 2.0.0
+
+CONTENTS
+1. What is Apache Flume Hadoop
+2. Major changes in this Release
+3. How to Get Involved
+4. How to Report Issues
+
+1. What is Apache Flume Hadoop
+Flume is a distributed, reliable, and available service for
+efficiently collecting, aggregating, and moving large amounts of event
+data. Flume Hadoop allows Flume to send data to Hadoop.
+
+2. Major changes in this Release
+For a detailed list of changes, please see the CHANGELOG file included
+in this distribution.
+
+4. How to Get Involved
+The Apache Flume project really needs and appreciates any contributions,
+including documentation help, source code and feedback. If you are interested
+in contributing, please visit:
+https://cwiki.apache.org/confluence/display/FLUME/How+to+Contribute
+
+5. How to Report Issues
+The Apache Flume Spring Boot project uses GitHub issues for issue tracking. Please see
+https://github.com/apache/flume-Hadoop/issues
diff --git a/checkstyle-header.txt b/checkstyle-header.txt
new file mode 100644
index 0000000..4f33236
--- /dev/null
+++ b/checkstyle-header.txt
@@ -0,0 +1,16 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache license, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the license for the specific language governing permissions and
+ * limitations under the license.
+ */
diff --git a/findbugs-exclude-filter.xml b/findbugs-exclude-filter.xml
new file mode 100644
index 0000000..327be31
--- /dev/null
+++ b/findbugs-exclude-filter.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- ===================================================================== -->
+<!-- $Id: findbugs-exclude-filter.xml 773234 2009-05-09 15:27:59Z rgoers $ -->
+<!-- ===================================================================== -->
+<FindBugsFilter>
+ <!-- Enable only high priority warnings -->
+ <Match>
+ <Priority value="2"/>
+ </Match>
+
+ <Match>
+ <Priority value="3"/>
+ </Match>
+</FindBugsFilter>
diff --git a/flume-hadoop-credential-store-config-filter/pom.xml b/flume-hadoop-credential-store-config-filter/pom.xml
new file mode 100644
index 0000000..7606298
--- /dev/null
+++ b/flume-hadoop-credential-store-config-filter/pom.xml
@@ -0,0 +1,59 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>flume-hadoop-parent</artifactId>
+ <groupId>org.apache.flume</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <properties>
+ <module.name>org.apache.flume.configfilter.credstore</module.name>
+ </properties>
+
+ <artifactId>flume-hadoop-credential-store-config-filter</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-config-filter-api</artifactId>
+ </dependency>
+ <!-- test dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.stefanbirkner</groupId>
+ <artifactId>system-rules</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+
+</project>
diff --git a/flume-hadoop-credential-store-config-filter/src/main/java/org/apache/flume/configfilter/HadoopCredentialStoreConfigFilter.java b/flume-hadoop-credential-store-config-filter/src/main/java/org/apache/flume/configfilter/HadoopCredentialStoreConfigFilter.java
new file mode 100644
index 0000000..c8d6870
--- /dev/null
+++ b/flume-hadoop-credential-store-config-filter/src/main/java/org/apache/flume/configfilter/HadoopCredentialStoreConfigFilter.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.flume.configfilter;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HadoopCredentialStoreConfigFilter extends AbstractConfigFilter {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(
+ HadoopCredentialStoreConfigFilter.class);
+
+ static final String PASSWORD_FILE_CONFIG_KEY
+ = "credstore.java-keystore-provider.password-file";
+ static final String CREDENTIAL_PROVIDER_PATH
+ = "credential.provider.path";
+ static final String HADOOP_SECURITY = "hadoop.security.";
+ private Configuration hadoopConfiguration;
+
+ public void initializeWithConfiguration(Map<String, String> configuration) {
+
+ LOGGER.debug("Initializing hadoop credential store.");
+ hadoopConfiguration = new Configuration();
+ hadoopConfiguration.set(
+ HADOOP_SECURITY + CREDENTIAL_PROVIDER_PATH,
+ configuration.get(CREDENTIAL_PROVIDER_PATH)
+ );
+
+ String passwordFile = configuration.get(PASSWORD_FILE_CONFIG_KEY);
+ if (passwordFile != null && !passwordFile.isEmpty()) {
+ checkPasswordFile(passwordFile);
+ hadoopConfiguration.set(
+ HADOOP_SECURITY + PASSWORD_FILE_CONFIG_KEY, passwordFile
+ );
+ }
+ }
+
+ private void checkPasswordFile(String passwordFile) {
+ if (Thread.currentThread().getContextClassLoader().getResource(passwordFile) == null) {
+ LOGGER.error("The java keystore provider password file has to be on the classpath." +
+ " The password file provided in the configuration cannot be found and will not be used"
+ );
+ }
+ }
+
+ @Override
+ public String filter(String key) {
+ char[] result = null;
+ try {
+ result = hadoopConfiguration.getPassword(key);
+ } catch (IOException e) {
+ LOGGER.error("Error while reading value for key {}: ", key, e);
+ }
+
+ return result == null ? null : new String(result);
+ }
+
+
+}
diff --git a/flume-hadoop-credential-store-config-filter/src/test/java/org/apache/flume/configfilter/TestHadoopCredentialStoreConfigFilter.java b/flume-hadoop-credential-store-config-filter/src/test/java/org/apache/flume/configfilter/TestHadoopCredentialStoreConfigFilter.java
new file mode 100644
index 0000000..55f74d8
--- /dev/null
+++ b/flume-hadoop-credential-store-config-filter/src/test/java/org/apache/flume/configfilter/TestHadoopCredentialStoreConfigFilter.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.flume.configfilter;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.alias.CredentialShell;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.contrib.java.lang.system.EnvironmentVariables;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.HashMap;
+
+import static org.apache.flume.configfilter.HadoopCredentialStoreConfigFilter.CREDENTIAL_PROVIDER_PATH;
+import static org.apache.flume.configfilter.HadoopCredentialStoreConfigFilter.HADOOP_SECURITY;
+import static org.apache.flume.configfilter.HadoopCredentialStoreConfigFilter.PASSWORD_FILE_CONFIG_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+public class TestHadoopCredentialStoreConfigFilter {
+
+ private static String providerPathDefault;
+ private static String providerPathEnv;
+ private static String providerPathPwdFile;
+
+ @ClassRule
+ public static final EnvironmentVariables environmentVariables
+ = new EnvironmentVariables();
+ private static File fileDefault;
+ private static File fileEnvPassword;
+ private static File fileFilePassword;
+ private HadoopCredentialStoreConfigFilter configFilter;
+
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+ generateTempFileNames();
+ fillCredStoreWithDefaultPassword();
+ fillCredStoreWithPasswordFile();
+ fillCredStoreWithEnvironmentVariablePassword();
+ }
+ @AfterClass
+ public static void tearDown() {
+ fileDefault.deleteOnExit();
+ fileEnvPassword.deleteOnExit();
+ fileFilePassword.deleteOnExit();
+ }
+
+ @Before
+ public void setUp() {
+ String[] objects = System.getenv().keySet().toArray(new String[0]);
+ environmentVariables.clear(objects);
+ configFilter = new HadoopCredentialStoreConfigFilter();
+ }
+
+ @Test
+ public void filterDefaultPasswordFile() {
+ HashMap<String, String> configuration = new HashMap<>();
+ configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathDefault);
+ configFilter.initializeWithConfiguration(configuration);
+
+ assertEquals("filtered_default", configFilter.filter("password"));
+ }
+
+ @Test
+ public void filterWithEnvPassword() {
+ environmentVariables.set("HADOOP_CREDSTORE_PASSWORD","envSecret");
+ HashMap<String, String> configuration = new HashMap<>();
+ configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathEnv);
+ configFilter.initializeWithConfiguration(configuration);
+
+ assertEquals("filtered_env", configFilter.filter("password"));
+ }
+
+ @Test
+ public void filterWithPasswordFile() {
+ HashMap<String, String> configuration = new HashMap<>();
+ configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathPwdFile);
+ configuration.put(PASSWORD_FILE_CONFIG_KEY, "test-password.txt");
+ configFilter.initializeWithConfiguration(configuration);
+
+ assertEquals("filtered_file", configFilter.filter("password"));
+ }
+
+ @Test
+ public void filterWithEnvNoPassword() {
+ HashMap<String, String> configuration = new HashMap<>();
+ configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathEnv);
+ configFilter.initializeWithConfiguration(configuration);
+
+ assertNull(configFilter.filter("password"));
+ }
+
+ @Test
+ public void filterErrorWithPasswordFileWrongPassword() {
+ HashMap<String, String> configuration = new HashMap<>();
+ configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathPwdFile);
+ configuration.put(PASSWORD_FILE_CONFIG_KEY, "test-password2.txt");
+ configFilter.initializeWithConfiguration(configuration);
+
+ assertNull(configFilter.filter("password"));
+ }
+
+ @Test
+ public void filterErrorWithPasswordFileNoPasswordFile() {
+ HashMap<String, String> configuration = new HashMap<>();
+ configuration.put(CREDENTIAL_PROVIDER_PATH, providerPathPwdFile);
+ configFilter.initializeWithConfiguration(configuration);
+
+ assertNull(configFilter.filter("password"));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void filterErrorWithNoProvider() {
+ HashMap<String, String> configuration = new HashMap<>();
+ configFilter.initializeWithConfiguration(configuration);
+ }
+
+
+ private static void fillCredStoreWithEnvironmentVariablePassword() throws Exception {
+ environmentVariables.set("HADOOP_CREDSTORE_PASSWORD", "envSecret");
+
+ runCommand("create password -value filtered_env -provider "
+ + providerPathEnv, new Configuration());
+ }
+
+ private static void fillCredStoreWithPasswordFile() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(
+ HADOOP_SECURITY + PASSWORD_FILE_CONFIG_KEY,
+ "test-password.txt"
+ );
+ runCommand("create password -value filtered_file -provider "
+ + providerPathPwdFile, conf);
+ }
+
+ private static void fillCredStoreWithDefaultPassword() throws Exception {
+ runCommand("create password -value filtered_default -provider "
+ + providerPathDefault, new Configuration());
+ }
+
+ private static void generateTempFileNames() throws IOException {
+ fileDefault = Files.createTempFile("test-default-pwd-", ".jceks").toFile();
+ boolean deleted = fileDefault.delete();
+ fileEnvPassword = Files.createTempFile("test-env-pwd-", ".jceks").toFile();
+ deleted &= fileEnvPassword.delete();
+ fileFilePassword = Files.createTempFile("test-file-pwd-", ".jceks").toFile();
+ deleted &= fileFilePassword.delete();
+ if (!deleted) {
+ fail("Could not delete temporary files");
+ }
+
+ providerPathDefault = "jceks://file/" + fileDefault.getAbsolutePath();
+ providerPathEnv = "jceks://file/" + fileEnvPassword.getAbsolutePath();
+ providerPathPwdFile = "jceks://file/" + fileFilePassword.getAbsolutePath();
+ }
+
+ private static void runCommand(String c, Configuration conf) throws Exception {
+ ToolRunner.run(conf, new CredentialShell(), c.split(" "));
+ }
+
+
+}
\ No newline at end of file
diff --git a/flume-hadoop-credential-store-config-filter/src/test/resources/test-password.txt b/flume-hadoop-credential-store-config-filter/src/test/resources/test-password.txt
new file mode 100644
index 0000000..db5fd91
--- /dev/null
+++ b/flume-hadoop-credential-store-config-filter/src/test/resources/test-password.txt
@@ -0,0 +1 @@
+fileSecret
\ No newline at end of file
diff --git a/flume-hadoop-credential-store-config-filter/src/test/resources/test-password2.txt b/flume-hadoop-credential-store-config-filter/src/test/resources/test-password2.txt
new file mode 100644
index 0000000..2dee0b1
--- /dev/null
+++ b/flume-hadoop-credential-store-config-filter/src/test/resources/test-password2.txt
@@ -0,0 +1 @@
+fileSecret2
\ No newline at end of file
diff --git a/flume-hadoop-dist/pom.xml b/flume-hadoop-dist/pom.xml
new file mode 100644
index 0000000..a046232
--- /dev/null
+++ b/flume-hadoop-dist/pom.xml
@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-hadoop-parent</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>flume-hadoop-dist</artifactId>
+ <name>Flume Hadoop Distribution</name>
+ <packaging>pom</packaging>
+
+ <properties>
+ <maven.deploy.skip>true</maven.deploy.skip>
+ <maven.install.skip>true</maven.install.skip>
+ <maven.test.skip>true</maven.test.skip>
+ <spotless.check.skip>true</spotless.check.skip>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-hadoop-credential-store-config-filter</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-hbase2-sink</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-hdfs-sink</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-hive-sink</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-kudu-sink</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <!-- calculate checksums of source release for Apache dist area -->
+ <plugin>
+ <groupId>net.nicoulaj.maven.plugins</groupId>
+ <artifactId>checksum-maven-plugin</artifactId>
+ <version>${checksum-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>calculate-checksums</id>
+ <goals>
+ <goal>files</goal>
+ </goals>
+ <!-- execute prior to maven-gpg-plugin:sign due to https://github.com/nicoulaj/checksum-maven-plugin/issues/112 -->
+ <phase>post-integration-test</phase>
+ <configuration>
+ <algorithms>
+ <algorithm>SHA-512</algorithm>
+ </algorithms>
+ <!-- https://maven.apache.org/apache-resource-bundles/#source-release-assembly-descriptor -->
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.directory}</directory>
+ <includes>
+ <include>apache-flume-hadoop-${project.version}-src.zip</include>
+ <include>apache-flume-hadoop-${project.version}-src.tar.gz</include>
+ <include>apache-flume-hadoop-${project.version}-bin.zip</include>
+ <include>apache-flume-hadoop-${project.version}-bin.tar.gz</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+ <csvSummary>false</csvSummary>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>post-integration-test</phase>
+ <configuration>
+ <target>
+ <property name="spaces" value=" " />
+ <concat destfile="${project.build.directory}/apache-flume-hadoop-${project.version}-src.zip.sha512" append="yes">${spaces}apache-flume-hadoop-${project.version}-src.zip</concat>
+ <concat destfile="${project.build.directory}/apache-flume-hadoop-${project.version}-src.tar.gz.sha512" append="yes">${spaces}apache-flume-hadoop-${project.version}-src.tar.gz</concat>
+ <concat destfile="${project.build.directory}/apache-flume-hadoop-${project.version}-bin.zip.sha512" append="yes">${spaces}apache-flume-hadoop-${project.version}-bin.zip</concat>
+ <concat destfile="${project.build.directory}/apache-flume-hadoop-${project.version}-bin.tar.gz.sha512" append="yes">${spaces}apache-flume-hadoop-${project.version}-bin.tar.gz</concat>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>source-release-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>apache-flume-hadoop-${project.version}</finalName>
+ <descriptors>
+ <descriptor>src/assembly/src.xml</descriptor>
+ </descriptors>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ </configuration>
+ </execution>
+ <execution>
+ <id>binary</id>
+ <configuration>
+ <finalName>apache-flume-hadoop-${project.version}</finalName>
+ <descriptors>
+ <descriptor>src/assembly/bin.xml</descriptor>
+ </descriptors>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ </configuration>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <phase>package</phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
diff --git a/flume-hadoop-dist/src/assembly/bin.xml b/flume-hadoop-dist/src/assembly/bin.xml
new file mode 100644
index 0000000..93fa8d7
--- /dev/null
+++ b/flume-hadoop-dist/src/assembly/bin.xml
@@ -0,0 +1,54 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<assembly>
+ <id>bin</id>
+ <formats>
+ <format>tar.gz</format>
+ <format>zip</format>
+ </formats>
+ <baseDirectory>apache-flume-twitter-${project.version}-bin</baseDirectory>
+ <includeSiteDirectory>false</includeSiteDirectory>
+ <moduleSets>
+ <moduleSet>
+ <useAllReactorProjects>true</useAllReactorProjects>
+ </moduleSet>
+ </moduleSets>
+ <dependencySets>
+ <dependencySet>
+ <includes>
+ <include>org.apache.flume:flume-hadoop-credential-store-config-filter</include>
+ <include>org.apache.flume:flume-hbase2-sink</include>
+ <include>org.apache.flume:flume-hdfs-sink</include>
+ <include>org.apache.flume:flume-hive-sink</include>
+ <include>org.apache.flume:flume-kudu-sink</include>
+ </includes>
+ <outputDirectory></outputDirectory>
+ <unpack>false</unpack>
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>../</directory>
+ <includes>
+ <include>LICENSE.txt</include>
+ <include>NOTICE.txt</include>
+ <include>RELEASE-NOTES.txt</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/flume-hadoop-dist/src/assembly/src.xml b/flume-hadoop-dist/src/assembly/src.xml
new file mode 100644
index 0000000..5873726
--- /dev/null
+++ b/flume-hadoop-dist/src/assembly/src.xml
@@ -0,0 +1,45 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<assembly>
+ <id>src</id>
+ <formats>
+ <format>zip</format>
+ <format>tar.gz</format>
+ </formats>
+ <baseDirectory>apache-flume-twitter-${project.version}-src</baseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>../</directory>
+
+ <excludes>
+ <exclude>**/target/**</exclude>
+ <exclude>**/.classpath</exclude>
+ <exclude>**/.project</exclude>
+ <exclude>**/.idea/**</exclude>
+ <exclude>**/*.iml</exclude>
+ <exclude>**/.settings/**</exclude>
+ <exclude>lib/**</exclude>
+ <exclude>**/.DS_Store</exclude>
+ <exclude>./mvn/wrapper/maven-wrapper.jar</exclude>
+ </excludes>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/flume-hbase2-sink/pom.xml b/flume-hbase2-sink/pom.xml
new file mode 100644
index 0000000..a5eb4ae
--- /dev/null
+++ b/flume-hbase2-sink/pom.xml
@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>flume-hadoop-parent</artifactId>
+ <groupId>org.apache.flume</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-hbase2-sink</artifactId>
+ <name>Flume HBase2 Sink</name>
+
+ <properties>
+ <!-- TODO fix spotbugs violations -->
+ <spotbugs.maxAllowedViolations>3</spotbugs.maxAllowedViolations>
+ <module.name>org.apache.flume.sink.hbase2</module.name>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-sdk</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-configuration</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-testing-util</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+</project>
diff --git a/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/BatchAware.java b/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/BatchAware.java
new file mode 100644
index 0000000..f184fec
--- /dev/null
+++ b/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/BatchAware.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hbase2;
+
+/**
+ * This interface allows for implementing HBase serializers that are aware of
+ * batching. {@link #onBatchStart()} is called at the beginning of each batch
+ * by the sink.
+ */
+public interface BatchAware {
+ void onBatchStart();
+}
diff --git a/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/HBase2EventSerializer.java b/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/HBase2EventSerializer.java
new file mode 100644
index 0000000..3fb64a4
--- /dev/null
+++ b/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/HBase2EventSerializer.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hbase2;
+
+import java.util.List;
+
+import org.apache.flume.Event;
+import org.apache.flume.conf.Configurable;
+import org.apache.flume.conf.ConfigurableComponent;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Row;
+
+/**
+ * Interface for an event serializer which serializes the headers and body
+ * of an event to write them to HBase 2. This is configurable, so any config
+ * params required should be taken through this. Only the column family is
+ * passed in. The columns should exist in the table and column family
+ * specified in the configuration for the HBase2Sink.
+ */
+public interface HBase2EventSerializer extends Configurable, ConfigurableComponent {
+ /**
+ * Initialize the event serializer.
+ * @param event Event to be written to HBase
+ * @param columnFamily Column family to write to
+ */
+ void initialize(Event event, byte[] columnFamily);
+
+ /**
+ * Get the actions that should be written out to hbase as a result of this
+ * event. This list is written to HBase using the HBase batch API.
+ * @return List of {@link org.apache.hadoop.hbase.client.Row} which
+ * are written as such to HBase.
+ *
+ * 0.92 increments do not implement Row, so this is not generic.
+ *
+ */
+ List<Row> getActions();
+
+ List<Increment> getIncrements();
+
+ /*
+ * Clean up any state. This will be called when the sink is being stopped.
+ */
+ void close();
+}
diff --git a/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/HBase2Sink.java b/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/HBase2Sink.java
new file mode 100644
index 0000000..42db0be
--- /dev/null
+++ b/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/HBase2Sink.java
@@ -0,0 +1,555 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hbase2;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.flume.Channel;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.EventDeliveryException;
+import org.apache.flume.FlumeException;
+import org.apache.flume.Transaction;
+import org.apache.flume.annotations.InterfaceAudience;
+import org.apache.flume.auth.FlumeAuthenticationUtil;
+import org.apache.flume.auth.PrivilegedExecutor;
+import org.apache.flume.conf.BatchSizeSupported;
+import org.apache.flume.conf.Configurable;
+import org.apache.flume.conf.ConfigurationException;
+import org.apache.flume.instrumentation.SinkCounter;
+import org.apache.flume.sink.AbstractSink;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.VersionInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+
+/**
+ * A simple sink which reads events from a channel and writes them to HBase 2.
+ * The HBase configuration is picked up from the first <tt>hbase-site.xml</tt>
+ * encountered in the classpath. This sink supports batch reading of
+ * events from the channel, and writing them to HBase, to minimize the number
+ * of flushes on the HBase tables. To use this sink, it has to be configured
+ * with certain mandatory parameters:<p>
+ * <tt>table: </tt> The name of the table in HBase to write to. <p>
+ * <tt>columnFamily: </tt> The column family in HBase to write to.<p>
+ * This sink will commit each transaction if the table's write buffer size is
+ * reached or if the number of events in the current transaction reaches the
+ * batch size, whichever comes first.<p>
+ * Other optional parameters are:<p>
+ * <tt>serializer:</tt> A class implementing {@link HBase2EventSerializer}.
+ * An instance of
+ * this class will be used to write out events to HBase.<p>
+ * <tt>serializer.*:</tt> Passed in the configure() method to serializer
+ * as an object of {@link org.apache.flume.Context}.<p>
+ * <tt>batchSize: </tt>This is the batch size used by the client. This is the
+ * maximum number of events the sink will commit per transaction. The default
+ * batch size is 100 events.
+ * <p>
+ * <p>
+ * <strong>Note: </strong> While this sink flushes all events in a transaction
+ * to HBase in one shot, HBase does not guarantee atomic commits on multiple
+ * rows. So if a subset of events in a batch are written to disk by HBase and
+ * HBase fails, the flume transaction is rolled back, causing flume to write
+ * all the events in the transaction all over again, which will cause
+ * duplicates. The serializer is expected to take care of the handling of
+ * duplicates etc. HBase also does not support batch increments, so if
+ * multiple increments are returned by the serializer, then HBase failure
+ * will cause them to be re-written, when HBase comes back up.
+ */
+public class HBase2Sink extends AbstractSink implements Configurable, BatchSizeSupported {
+ private String tableName;
+ private byte[] columnFamily;
+ private Connection conn;
+ private BufferedMutator table;
+ private long batchSize;
+ private final Configuration config;
+ private static final Logger logger = LoggerFactory.getLogger(HBase2Sink.class);
+ private HBase2EventSerializer serializer;
+ private String kerberosPrincipal;
+ private String kerberosKeytab;
+ private boolean enableWal = true;
+ private boolean batchIncrements = false;
+ private SinkCounter sinkCounter;
+ private PrivilegedExecutor privilegedExecutor;
+
+ // Internal hooks used for unit testing.
+ private DebugIncrementsCallback debugIncrCallback = null;
+
+ public HBase2Sink() {
+ this(HBaseConfiguration.create());
+ }
+
+ public HBase2Sink(Configuration conf) {
+ this.config = conf;
+ }
+
+ @VisibleForTesting
+ @InterfaceAudience.Private
+ HBase2Sink(Configuration conf, DebugIncrementsCallback cb) {
+ this(conf);
+ this.debugIncrCallback = cb;
+ }
+
+ @Override
+ public void start() {
+ Preconditions.checkArgument(table == null, "Please call stop " +
+ "before calling start on an old instance.");
+ try {
+ privilegedExecutor =
+ FlumeAuthenticationUtil.getAuthenticator(kerberosPrincipal, kerberosKeytab);
+ } catch (Exception ex) {
+ sinkCounter.incrementConnectionFailedCount();
+ throw new FlumeException("Failed to login to HBase using "
+ + "provided credentials.", ex);
+ }
+ try {
+ conn = privilegedExecutor.execute((PrivilegedExceptionAction<Connection>) () -> {
+ conn = ConnectionFactory.createConnection(config);
+ return conn;
+ });
+ // Flush is controlled by us. This ensures that HBase changing
+ // their criteria for flushing does not change how we flush.
+ table = conn.getBufferedMutator(TableName.valueOf(tableName));
+
+ } catch (Exception e) {
+ sinkCounter.incrementConnectionFailedCount();
+ logger.error("Could not load table, " + tableName +
+ " from HBase", e);
+ throw new FlumeException("Could not load table, " + tableName +
+ " from HBase", e);
+ }
+ try {
+ if (!privilegedExecutor.execute((PrivilegedExceptionAction<Boolean>) () -> {
+ Table t = null;
+ try {
+ t = conn.getTable(TableName.valueOf(tableName));
+ return t.getTableDescriptor().hasFamily(columnFamily);
+ } finally {
+ if (t != null) {
+ t.close();
+ }
+ }
+ })) {
+ throw new IOException("Table " + tableName
+ + " has no such column family " + Bytes.toString(columnFamily));
+ }
+ } catch (Exception e) {
+ //Get getTableDescriptor also throws IOException, so catch the IOException
+ //thrown above or by the getTableDescriptor() call.
+ sinkCounter.incrementConnectionFailedCount();
+ throw new FlumeException("Error getting column family from HBase."
+ + "Please verify that the table " + tableName + " and Column Family, "
+ + Bytes.toString(columnFamily) + " exists in HBase, and the"
+ + " current user has permissions to access that table.", e);
+ }
+
+ super.start();
+ sinkCounter.incrementConnectionCreatedCount();
+ sinkCounter.start();
+ }
+
+ @Override
+ public void stop() {
+ try {
+ if (table != null) {
+ table.close();
+ }
+ table = null;
+ } catch (IOException e) {
+ throw new FlumeException("Error closing table.", e);
+ }
+ try {
+ if (conn != null) {
+ conn.close();
+ }
+ conn = null;
+ } catch (IOException e) {
+ throw new FlumeException("Error closing connection.", e);
+ }
+ sinkCounter.incrementConnectionClosedCount();
+ sinkCounter.stop();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void configure(Context context) {
+ if (!this.hasVersionAtLeast2()) {
+ throw new ConfigurationException(
+ "HBase major version number must be at least 2 for hbase2sink");
+ }
+
+ tableName = context.getString(HBase2SinkConfigurationConstants.CONFIG_TABLE);
+ String cf = context.getString(
+ HBase2SinkConfigurationConstants.CONFIG_COLUMN_FAMILY);
+ batchSize = context.getLong(
+ HBase2SinkConfigurationConstants.CONFIG_BATCHSIZE, 100L);
+ Context serializerContext = new Context();
+ //If not specified, will use HBase defaults.
+ String eventSerializerType = context.getString(
+ HBase2SinkConfigurationConstants.CONFIG_SERIALIZER);
+ Preconditions.checkNotNull(tableName,
+ "Table name cannot be empty, please specify in configuration file");
+ Preconditions.checkNotNull(cf,
+ "Column family cannot be empty, please specify in configuration file");
+ //Check foe event serializer, if null set event serializer type
+ if (eventSerializerType == null || eventSerializerType.isEmpty()) {
+ eventSerializerType =
+ "org.apache.flume.sink.hbase2.SimpleHBase2EventSerializer";
+ logger.info("No serializer defined, Will use default");
+ }
+ serializerContext.putAll(context.getSubProperties(
+ HBase2SinkConfigurationConstants.CONFIG_SERIALIZER_PREFIX));
+ columnFamily = cf.getBytes(Charsets.UTF_8);
+ try {
+ Class<? extends HBase2EventSerializer> clazz =
+ (Class<? extends HBase2EventSerializer>)
+ Class.forName(eventSerializerType);
+ serializer = clazz.newInstance();
+ serializer.configure(serializerContext);
+ } catch (Exception e) {
+ logger.error("Could not instantiate event serializer.", e);
+ Throwables.propagate(e);
+ }
+ kerberosKeytab = context.getString(HBase2SinkConfigurationConstants.CONFIG_KEYTAB);
+ kerberosPrincipal = context.getString(HBase2SinkConfigurationConstants.CONFIG_PRINCIPAL);
+
+ enableWal = context.getBoolean(HBase2SinkConfigurationConstants
+ .CONFIG_ENABLE_WAL, HBase2SinkConfigurationConstants.DEFAULT_ENABLE_WAL);
+ logger.info("The write to WAL option is set to: " + String.valueOf(enableWal));
+ if (!enableWal) {
+ logger.warn("HBase Sink's enableWal configuration is set to false. All " +
+ "writes to HBase will have WAL disabled, and any data in the " +
+ "memstore of this region in the Region Server could be lost!");
+ }
+
+ batchIncrements = context.getBoolean(
+ HBase2SinkConfigurationConstants.CONFIG_COALESCE_INCREMENTS,
+ HBase2SinkConfigurationConstants.DEFAULT_COALESCE_INCREMENTS);
+
+ if (batchIncrements) {
+ logger.info("Increment coalescing is enabled. Increments will be " +
+ "buffered.");
+ }
+
+ String zkQuorum = context.getString(HBase2SinkConfigurationConstants
+ .ZK_QUORUM);
+ Integer port = null;
+ /*
+ * HBase allows multiple nodes in the quorum, but all need to use the
+ * same client port. So get the nodes in host:port format,
+ * and ignore the ports for all nodes except the first one. If no port is
+ * specified, use default.
+ */
+ if (zkQuorum != null && !zkQuorum.isEmpty()) {
+ StringBuilder zkBuilder = new StringBuilder();
+ logger.info("Using ZK Quorum: " + zkQuorum);
+ String[] zkHosts = zkQuorum.split(",");
+ int length = zkHosts.length;
+ for (int i = 0; i < length; i++) {
+ String[] zkHostAndPort = zkHosts[i].split(":");
+ zkBuilder.append(zkHostAndPort[0].trim());
+ if (i != length - 1) {
+ zkBuilder.append(",");
+ } else {
+ zkQuorum = zkBuilder.toString();
+ }
+ if (zkHostAndPort[1] == null) {
+ throw new FlumeException("Expected client port for the ZK node!");
+ }
+ if (port == null) {
+ port = Integer.parseInt(zkHostAndPort[1].trim());
+ } else if (!port.equals(Integer.parseInt(zkHostAndPort[1].trim()))) {
+ throw new FlumeException("All Zookeeper nodes in the quorum must " +
+ "use the same client port.");
+ }
+ }
+ if (port == null) {
+ port = HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT;
+ }
+ this.config.set(HConstants.ZOOKEEPER_QUORUM, zkQuorum);
+ this.config.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, port);
+ }
+ String hbaseZnode = context.getString(
+ HBase2SinkConfigurationConstants.ZK_ZNODE_PARENT);
+ if (hbaseZnode != null && !hbaseZnode.isEmpty()) {
+ this.config.set(HConstants.ZOOKEEPER_ZNODE_PARENT, hbaseZnode);
+ }
+ sinkCounter = new SinkCounter(this.getName());
+ }
+
+ public Configuration getConfig() {
+ return config;
+ }
+
+ @Override
+ public Status process() throws EventDeliveryException {
+ Status status = Status.READY;
+ Channel channel = getChannel();
+ Transaction txn = channel.getTransaction();
+ List<Row> actions = new LinkedList<>();
+ List<Increment> incs = new LinkedList<>();
+ try {
+ txn.begin();
+
+ if (serializer instanceof BatchAware) {
+ ((BatchAware) serializer).onBatchStart();
+ }
+
+ long i = 0;
+ for (; i < batchSize; i++) {
+ Event event = channel.take();
+ if (event == null) {
+ if (i == 0) {
+ status = Status.BACKOFF;
+ sinkCounter.incrementBatchEmptyCount();
+ } else {
+ sinkCounter.incrementBatchUnderflowCount();
+ }
+ break;
+ } else {
+ serializer.initialize(event, columnFamily);
+ actions.addAll(serializer.getActions());
+ incs.addAll(serializer.getIncrements());
+ }
+ }
+ if (i == batchSize) {
+ sinkCounter.incrementBatchCompleteCount();
+ }
+ sinkCounter.addToEventDrainAttemptCount(i);
+
+ putEventsAndCommit(actions, incs, txn);
+
+ } catch (Throwable e) {
+ try {
+ txn.rollback();
+ } catch (Exception e2) {
+ logger.error("Exception in rollback. Rollback might not have been " +
+ "successful.", e2);
+ }
+ logger.error("Failed to commit transaction." +
+ "Transaction rolled back.", e);
+ sinkCounter.incrementEventWriteOrChannelFail(e);
+ if (e instanceof Error || e instanceof RuntimeException) {
+ logger.error("Failed to commit transaction." +
+ "Transaction rolled back.", e);
+ Throwables.propagate(e);
+ } else {
+ logger.error("Failed to commit transaction." +
+ "Transaction rolled back.", e);
+ throw new EventDeliveryException("Failed to commit transaction." +
+ "Transaction rolled back.", e);
+ }
+ } finally {
+ txn.close();
+ }
+ return status;
+ }
+
+ private void putEventsAndCommit(final List<Row> actions,
+ final List<Increment> incs, Transaction txn) throws Exception {
+
+ privilegedExecutor.execute((PrivilegedExceptionAction<Void>) () -> {
+ final List<Mutation> mutations = new ArrayList<>(actions.size());
+ for (Row r : actions) {
+ if (r instanceof Put) {
+ ((Put) r).setDurability(enableWal ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
+ }
+ // Newer versions of HBase - Increment implements Row.
+ if (r instanceof Increment) {
+ ((Increment) r).setDurability(enableWal ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
+ }
+ if (r instanceof Mutation) {
+ mutations.add((Mutation)r);
+ } else {
+ logger.warn("dropping row " + r + " since it is not an Increment or Put");
+ }
+ }
+ table.mutate(mutations);
+ table.flush();
+ return null;
+ });
+
+ privilegedExecutor.execute((PrivilegedExceptionAction<Void>) () -> {
+
+ List<Increment> processedIncrements;
+ if (batchIncrements) {
+ processedIncrements = coalesceIncrements(incs);
+ } else {
+ processedIncrements = incs;
+ }
+
+ // Only used for unit testing.
+ if (debugIncrCallback != null) {
+ debugIncrCallback.onAfterCoalesce(processedIncrements);
+ }
+
+ for (final Increment i : processedIncrements) {
+ i.setDurability(enableWal ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
+ table.mutate(i);
+ }
+ table.flush();
+ return null;
+ });
+
+ txn.commit();
+ sinkCounter.addToEventDrainSuccessCount(actions.size());
+ }
+
+
+ @SuppressWarnings("unchecked")
+ private Map<byte[], NavigableMap<byte[], Long>> getFamilyMap(Increment inc) {
+ Preconditions.checkNotNull(inc, "Increment required");
+ return inc.getFamilyMapOfLongs();
+ }
+
+ /**
+ * Perform "compression" on the given set of increments so that Flume sends
+ * the minimum possible number of RPC operations to HBase per batch.
+ *
+ * @param incs Input: Increment objects to coalesce.
+ * @return List of new Increment objects after coalescing the unique counts.
+ */
+ private List<Increment> coalesceIncrements(Iterable<Increment> incs) {
+ Preconditions.checkNotNull(incs, "List of Increments must not be null");
+ // Aggregate all of the increment row/family/column counts.
+ // The nested map is keyed like this: {row, family, qualifier} => count.
+ Map<byte[], Map<byte[], NavigableMap<byte[], Long>>> counters =
+ Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ for (Increment inc : incs) {
+ byte[] row = inc.getRow();
+ Map<byte[], NavigableMap<byte[], Long>> families = getFamilyMap(inc);
+ for (Map.Entry<byte[], NavigableMap<byte[], Long>> familyEntry : families.entrySet()) {
+ byte[] family = familyEntry.getKey();
+ NavigableMap<byte[], Long> qualifiers = familyEntry.getValue();
+ for (Map.Entry<byte[], Long> qualifierEntry : qualifiers.entrySet()) {
+ byte[] qualifier = qualifierEntry.getKey();
+ Long count = qualifierEntry.getValue();
+ incrementCounter(counters, row, family, qualifier, count);
+ }
+ }
+ }
+
+ // Reconstruct list of Increments per unique row/family/qualifier.
+ List<Increment> coalesced = Lists.newLinkedList();
+ for (Map.Entry<byte[], Map<byte[], NavigableMap<byte[], Long>>> rowEntry :
+ counters.entrySet()) {
+ byte[] row = rowEntry.getKey();
+ Map<byte[], NavigableMap<byte[], Long>> families = rowEntry.getValue();
+ Increment inc = new Increment(row);
+ for (Map.Entry<byte[], NavigableMap<byte[], Long>> familyEntry : families.entrySet()) {
+ byte[] family = familyEntry.getKey();
+ NavigableMap<byte[], Long> qualifiers = familyEntry.getValue();
+ for (Map.Entry<byte[], Long> qualifierEntry : qualifiers.entrySet()) {
+ byte[] qualifier = qualifierEntry.getKey();
+ long count = qualifierEntry.getValue();
+ inc.addColumn(family, qualifier, count);
+ }
+ }
+ coalesced.add(inc);
+ }
+
+ return coalesced;
+ }
+
+ /**
+ * Helper function for {@link #coalesceIncrements} to increment a counter
+ * value in the passed data structure.
+ *
+ * @param counters Nested data structure containing the counters.
+ * @param row Row key to increment.
+ * @param family Column family to increment.
+ * @param qualifier Column qualifier to increment.
+ * @param count Amount to increment by.
+ */
+ private void incrementCounter(
+ Map<byte[], Map<byte[], NavigableMap<byte[], Long>>> counters,
+ byte[] row, byte[] family, byte[] qualifier, Long count) {
+
+ Map<byte[], NavigableMap<byte[], Long>> families =
+ counters.computeIfAbsent(row, k -> Maps.newTreeMap(Bytes.BYTES_COMPARATOR));
+
+ NavigableMap<byte[], Long> qualifiers =
+ families.computeIfAbsent(family, k -> Maps.newTreeMap(Bytes.BYTES_COMPARATOR));
+
+ qualifiers.merge(qualifier, count, (a, b) -> a + b);
+ }
+
+ String getHBbaseVersionString() {
+ return VersionInfo.getVersion();
+ }
+
+ private int getMajorVersion(String version) throws NumberFormatException {
+ return Integer.parseInt(version.split("\\.")[0]);
+ }
+
+ private boolean hasVersionAtLeast2() {
+ String version = getHBbaseVersionString();
+ try {
+ if (this.getMajorVersion(version) >= 2) {
+ return true;
+ }
+ } catch (NumberFormatException ex) {
+ logger.error(ex.getMessage());
+ }
+ logger.error("Invalid HBase version for hbase2sink:" + version);
+ return false;
+ }
+
+ @VisibleForTesting
+ @InterfaceAudience.Private
+ HBase2EventSerializer getSerializer() {
+ return serializer;
+ }
+
+ @Override
+ public long getBatchSize() {
+ return batchSize;
+ }
+
+ @VisibleForTesting
+ @InterfaceAudience.Private
+ interface DebugIncrementsCallback {
+ void onAfterCoalesce(Iterable<Increment> increments);
+ }
+}
diff --git a/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/HBase2SinkConfigurationConstants.java b/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/HBase2SinkConfigurationConstants.java
new file mode 100644
index 0000000..15aa9e6
--- /dev/null
+++ b/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/HBase2SinkConfigurationConstants.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hbase2;
+
+import org.apache.hadoop.hbase.HConstants;
+
+/**
+ * Constants used for configuration of HBaseSink2
+ *
+ */
+public class HBase2SinkConfigurationConstants {
+ /**
+ * The HBase table which the sink should write to.
+ */
+ public static final String CONFIG_TABLE = "table";
+ /**
+ * The column family which the sink should use.
+ */
+ public static final String CONFIG_COLUMN_FAMILY = "columnFamily";
+ /**
+ * Maximum number of events the sink should take from the channel per
+ * transaction, if available.
+ */
+ public static final String CONFIG_BATCHSIZE = "batchSize";
+ /**
+ * The fully qualified class name of the serializer the sink should use.
+ */
+ public static final String CONFIG_SERIALIZER = "serializer";
+ /**
+ * Configuration to pass to the serializer.
+ */
+ public static final String CONFIG_SERIALIZER_PREFIX = CONFIG_SERIALIZER + ".";
+
+ public static final String CONFIG_TIMEOUT = "timeout";
+
+ public static final String CONFIG_ENABLE_WAL = "enableWal";
+
+ public static final boolean DEFAULT_ENABLE_WAL = true;
+
+ public static final long DEFAULT_TIMEOUT = 60000;
+
+ public static final String CONFIG_KEYTAB = "kerberosKeytab";
+
+ public static final String CONFIG_PRINCIPAL = "kerberosPrincipal";
+
+ public static final String ZK_QUORUM = "zookeeperQuorum";
+
+ public static final String ZK_ZNODE_PARENT = "znodeParent";
+
+ public static final String DEFAULT_ZK_ZNODE_PARENT =
+ HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT;
+
+ public static final String CONFIG_COALESCE_INCREMENTS = "coalesceIncrements";
+
+ public static final Boolean DEFAULT_COALESCE_INCREMENTS = false;
+
+ public static final int DEFAULT_MAX_CONSECUTIVE_FAILS = 10;
+
+ public static final String CONFIG_MAX_CONSECUTIVE_FAILS = "maxConsecutiveFails";
+
+}
diff --git a/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/RegexHBase2EventSerializer.java b/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/RegexHBase2EventSerializer.java
new file mode 100644
index 0000000..089795e
--- /dev/null
+++ b/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/RegexHBase2EventSerializer.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hbase2;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.FlumeException;
+import org.apache.flume.conf.ComponentConfiguration;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Row;
+
+import java.nio.charset.Charset;
+import java.util.Calendar;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * An {@link HBase2EventSerializer} which parses columns based on a supplied
+ * regular expression and column name list.
+ * <p>
+ * Note that if the regular expression does not return the correct number of
+ * groups for a particular event, or it does not correctly match an event,
+ * the event is silently dropped.
+ * <p>
+ * Row keys for each event consist of a timestamp concatenated with an
+ * identifier which enforces uniqueness of keys across flume agents.
+ * <p>
+ * See static constant variables for configuration options.
+ */
+public class RegexHBase2EventSerializer implements HBase2EventSerializer {
+ // Config vars
+ /** Regular expression used to parse groups from event data. */
+ public static final String REGEX_CONFIG = "regex";
+ public static final String REGEX_DEFAULT = "(.*)";
+
+ /** Whether to ignore case when performing regex matches. */
+ public static final String IGNORE_CASE_CONFIG = "regexIgnoreCase";
+ public static final boolean IGNORE_CASE_DEFAULT = false;
+
+ /** Comma separated list of column names to place matching groups in. */
+ public static final String COL_NAME_CONFIG = "colNames";
+ public static final String COLUMN_NAME_DEFAULT = "payload";
+
+ /** Index of the row key in matched regex groups */
+ public static final String ROW_KEY_INDEX_CONFIG = "rowKeyIndex";
+
+ /** Placeholder in colNames for row key */
+ public static final String ROW_KEY_NAME = "ROW_KEY";
+
+ /** Whether to deposit event headers into corresponding column qualifiers */
+ public static final String DEPOSIT_HEADERS_CONFIG = "depositHeaders";
+ public static final boolean DEPOSIT_HEADERS_DEFAULT = false;
+
+ /** What charset to use when serializing into HBase's byte arrays */
+ public static final String CHARSET_CONFIG = "charset";
+ public static final String CHARSET_DEFAULT = "UTF-8";
+
+ /* This is a nonce used in HBase row-keys, such that the same row-key
+ * never gets written more than once from within this JVM. */
+ protected static final AtomicInteger nonce = new AtomicInteger(0);
+ protected static final String randomKey = RandomStringUtils.randomAlphanumeric(10);
+
+ protected byte[] cf;
+ private byte[] payload;
+ private final List<byte[]> colNames = Lists.newArrayList();
+ private Map<String, String> headers;
+ private boolean depositHeaders;
+ private Pattern inputPattern;
+ private Charset charset;
+ private int rowKeyIndex;
+
+ @Override
+ public void configure(Context context) {
+ String regex = context.getString(REGEX_CONFIG, REGEX_DEFAULT);
+ boolean regexIgnoreCase = context.getBoolean(IGNORE_CASE_CONFIG,
+ IGNORE_CASE_DEFAULT);
+ depositHeaders = context.getBoolean(DEPOSIT_HEADERS_CONFIG,
+ DEPOSIT_HEADERS_DEFAULT);
+ inputPattern = Pattern.compile(regex, Pattern.DOTALL
+ + (regexIgnoreCase ? Pattern.CASE_INSENSITIVE : 0));
+ charset = Charset.forName(context.getString(CHARSET_CONFIG,
+ CHARSET_DEFAULT));
+
+ String colNameStr = context.getString(COL_NAME_CONFIG, COLUMN_NAME_DEFAULT);
+ String[] columnNames = colNameStr.split(",");
+ for (String s : columnNames) {
+ colNames.add(s.getBytes(charset));
+ }
+
+ //Rowkey is optional, default is -1
+ rowKeyIndex = context.getInteger(ROW_KEY_INDEX_CONFIG, -1);
+ //if row key is being used, make sure it is specified correct
+ if (rowKeyIndex >= 0) {
+ if (rowKeyIndex >= columnNames.length) {
+ throw new IllegalArgumentException(ROW_KEY_INDEX_CONFIG + " must be " +
+ "less than num columns " + columnNames.length);
+ }
+ if (!ROW_KEY_NAME.equalsIgnoreCase(columnNames[rowKeyIndex])) {
+ throw new IllegalArgumentException("Column at " + rowKeyIndex + " must be "
+ + ROW_KEY_NAME + " and is " + columnNames[rowKeyIndex]);
+ }
+ }
+ }
+
+ @Override
+ public void configure(ComponentConfiguration conf) {
+ }
+
+ @Override
+ public void initialize(Event event, byte[] columnFamily) {
+ this.headers = event.getHeaders();
+ this.payload = event.getBody();
+ this.cf = columnFamily;
+ }
+
+ /**
+ * Returns a row-key with the following format:
+ * [time in millis]-[random key]-[nonce]
+ */
+ protected byte[] getRowKey(Calendar cal) {
+ /* NOTE: This key generation strategy has the following properties:
+ *
+ * 1) Within a single JVM, the same row key will never be duplicated.
+ * 2) Amongst any two JVM's operating at different time periods (according
+ * to their respective clocks), the same row key will never be
+ * duplicated.
+ * 3) Amongst any two JVM's operating concurrently (according to their
+ * respective clocks), the odds of duplicating a row-key are non-zero
+ * but infinitesimal. This would require simultaneous collision in (a)
+ * the timestamp (b) the respective nonce and (c) the random string.
+ * The string is necessary since (a) and (b) could collide if a fleet
+ * of Flume agents are restarted in tandem.
+ *
+ * Row-key uniqueness is important because conflicting row-keys will cause
+ * data loss. */
+ String rowKey = String.format("%s-%s-%s", cal.getTimeInMillis(),
+ randomKey, nonce.getAndIncrement());
+ return rowKey.getBytes(charset);
+ }
+
+ protected byte[] getRowKey() {
+ return getRowKey(Calendar.getInstance());
+ }
+
+ @Override
+ public List<Row> getActions() throws FlumeException {
+ List<Row> actions = Lists.newArrayList();
+ byte[] rowKey;
+ Matcher m = inputPattern.matcher(new String(payload, charset));
+ if (!m.matches()) {
+ return Lists.newArrayList();
+ }
+
+ if (m.groupCount() != colNames.size()) {
+ return Lists.newArrayList();
+ }
+
+ try {
+ if (rowKeyIndex < 0) {
+ rowKey = getRowKey();
+ } else {
+ rowKey = m.group(rowKeyIndex + 1).getBytes(Charsets.UTF_8);
+ }
+ Put put = new Put(rowKey);
+
+ for (int i = 0; i < colNames.size(); i++) {
+ if (i != rowKeyIndex) {
+ put.addColumn(cf, colNames.get(i), m.group(i + 1).getBytes(Charsets.UTF_8));
+ }
+ }
+ if (depositHeaders) {
+ for (Map.Entry<String, String> entry : headers.entrySet()) {
+ put.addColumn(cf, entry.getKey().getBytes(charset), entry.getValue().getBytes(charset));
+ }
+ }
+ actions.add(put);
+ } catch (Exception e) {
+ throw new FlumeException("Could not get row key!", e);
+ }
+ return actions;
+ }
+
+ @Override
+ public List<Increment> getIncrements() {
+ return Lists.newArrayList();
+ }
+
+ @Override
+ public void close() {
+ }
+}
\ No newline at end of file
diff --git a/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/SimpleHBase2EventSerializer.java b/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/SimpleHBase2EventSerializer.java
new file mode 100644
index 0000000..8236543
--- /dev/null
+++ b/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/SimpleHBase2EventSerializer.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.flume.sink.hbase2;
+
+import com.google.common.base.Charsets;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.FlumeException;
+import org.apache.flume.conf.ComponentConfiguration;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Row;
+
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * A simple serializer that returns puts from an event, by writing the event
+ * body into it. The headers are discarded. It also updates a row in HBase
+ * which acts as an event counter.
+ * <p>Takes optional parameters:<p>
+ * <tt>rowPrefix:</tt> The prefix to be used. Default: <i>default</i><p>
+ * <tt>incrementRow</tt> The row to increment. Default: <i>incRow</i><p>
+ * <tt>suffix:</tt> <i>uuid/random/timestamp.</i>Default: <i>uuid</i><p>
+ * <p>Mandatory parameters: <p>
+ * <tt>cf:</tt>Column family.<p>
+ * Components that have no defaults and will not be used if null:
+ * <tt>payloadColumn:</tt> Which column to put payload in. If it is null,
+ * event data will not be written.<p>
+ * <tt>incColumn:</tt> Which column to increment. Null means no column is
+ * incremented.
+ */
+public class SimpleHBase2EventSerializer implements HBase2EventSerializer {
+ private String rowPrefix;
+ private byte[] incrementRow;
+ private byte[] cf;
+ private byte[] plCol;
+ private byte[] incCol;
+ private KeyType keyType;
+ private byte[] payload;
+
+ public SimpleHBase2EventSerializer() {
+ }
+
+ @Override
+ public void configure(Context context) {
+ rowPrefix = context.getString("rowPrefix", "default");
+ incrementRow =
+ context.getString("incrementRow", "incRow").getBytes(Charsets.UTF_8);
+ String suffix = context.getString("suffix", "uuid");
+
+ String payloadColumn = context.getString("payloadColumn", "pCol");
+ String incColumn = context.getString("incrementColumn", "iCol");
+ if (payloadColumn != null && !payloadColumn.isEmpty()) {
+ switch (suffix) {
+ case "timestamp":
+ keyType = KeyType.TS;
+ break;
+ case "random":
+ keyType = KeyType.RANDOM;
+ break;
+ case "nano":
+ keyType = KeyType.TSNANO;
+ break;
+ default:
+ keyType = KeyType.UUID;
+ break;
+ }
+ plCol = payloadColumn.getBytes(Charsets.UTF_8);
+ }
+ if (incColumn != null && !incColumn.isEmpty()) {
+ incCol = incColumn.getBytes(Charsets.UTF_8);
+ }
+ }
+
+ @Override
+ public void configure(ComponentConfiguration conf) {
+ }
+
+ @Override
+ public void initialize(Event event, byte[] cf) {
+ this.payload = event.getBody();
+ this.cf = cf;
+ }
+
+ @Override
+ public List<Row> getActions() throws FlumeException {
+ List<Row> actions = new LinkedList<>();
+ if (plCol != null) {
+ byte[] rowKey;
+ try {
+ if (keyType == KeyType.TS) {
+ rowKey = SimpleRowKeyGenerator.getTimestampKey(rowPrefix);
+ } else if (keyType == KeyType.RANDOM) {
+ rowKey = SimpleRowKeyGenerator.getRandomKey(rowPrefix);
+ } else if (keyType == KeyType.TSNANO) {
+ rowKey = SimpleRowKeyGenerator.getNanoTimestampKey(rowPrefix);
+ } else {
+ rowKey = SimpleRowKeyGenerator.getUUIDKey(rowPrefix);
+ }
+ Put put = new Put(rowKey);
+ put.addColumn(cf, plCol, payload);
+ actions.add(put);
+ } catch (Exception e) {
+ throw new FlumeException("Could not get row key!", e);
+ }
+
+ }
+ return actions;
+ }
+
+ @Override
+ public List<Increment> getIncrements() {
+ List<Increment> incs = new LinkedList<>();
+ if (incCol != null) {
+ Increment inc = new Increment(incrementRow);
+ inc.addColumn(cf, incCol, 1);
+ incs.add(inc);
+ }
+ return incs;
+ }
+
+ @Override
+ public void close() {
+ }
+
+ public enum KeyType {
+ UUID,
+ RANDOM,
+ TS,
+ TSNANO
+ }
+
+}
diff --git a/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/SimpleRowKeyGenerator.java b/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/SimpleRowKeyGenerator.java
new file mode 100644
index 0000000..5386107
--- /dev/null
+++ b/flume-hbase2-sink/src/main/java/org/apache/flume/sink/hbase2/SimpleRowKeyGenerator.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hbase2;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Random;
+import java.util.UUID;
+
+/**
+ * Utility class for users to generate their own keys. Any key can be used,
+ * this is just a utility that provides a set of simple keys.
+ */
+public class SimpleRowKeyGenerator {
+
+ public static byte[] getUUIDKey(String prefix) throws UnsupportedEncodingException {
+ return (prefix + UUID.randomUUID().toString()).getBytes("UTF8");
+ }
+
+ public static byte[] getRandomKey(String prefix) throws UnsupportedEncodingException {
+ return (prefix + String.valueOf(new Random().nextLong())).getBytes("UTF8");
+ }
+
+ public static byte[] getTimestampKey(String prefix) throws UnsupportedEncodingException {
+ return (prefix + String.valueOf(System.currentTimeMillis())).getBytes("UTF8");
+ }
+
+ public static byte[] getNanoTimestampKey(String prefix) throws UnsupportedEncodingException {
+ return (prefix + String.valueOf(System.nanoTime())).getBytes("UTF8");
+ }
+}
diff --git a/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/IncrementHBase2Serializer.java b/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/IncrementHBase2Serializer.java
new file mode 100644
index 0000000..f5112ae
--- /dev/null
+++ b/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/IncrementHBase2Serializer.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hbase2;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Charsets;
+import com.google.common.collect.Lists;
+import java.util.Collections;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.conf.ComponentConfiguration;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Row;
+
+import java.util.List;
+
+/**
+ * For Increment-related unit tests.
+ */
+class IncrementHBase2Serializer implements HBase2EventSerializer, BatchAware {
+ private Event event;
+ private byte[] family;
+ private int numBatchesStarted = 0;
+
+ @Override public void configure(Context context) {
+ }
+ @Override public void configure(ComponentConfiguration conf) {
+ }
+ @Override public void close() {
+ }
+
+ @Override
+ public void initialize(Event event, byte[] columnFamily) {
+ this.event = event;
+ this.family = columnFamily;
+ }
+
+ // This class only creates Increments.
+ @Override
+ public List<Row> getActions() {
+ return Collections.emptyList();
+ }
+
+ // Treat each Event as a String, i,e, "row:qualifier".
+ @Override
+ public List<Increment> getIncrements() {
+ List<Increment> increments = Lists.newArrayList();
+ String body = new String(event.getBody(), Charsets.UTF_8);
+ String[] pieces = body.split(":");
+ String row = pieces[0];
+ String qualifier = pieces[1];
+ Increment inc = new Increment(row.getBytes(Charsets.UTF_8));
+ inc.addColumn(family, qualifier.getBytes(Charsets.UTF_8), 1L);
+ increments.add(inc);
+ return increments;
+ }
+
+ @Override
+ public void onBatchStart() {
+ numBatchesStarted++;
+ }
+
+ @VisibleForTesting
+ public int getNumBatchesStarted() {
+ return numBatchesStarted;
+ }
+}
diff --git a/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/MockSimpleHBase2EventSerializer.java b/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/MockSimpleHBase2EventSerializer.java
new file mode 100644
index 0000000..6c6da71
--- /dev/null
+++ b/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/MockSimpleHBase2EventSerializer.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.flume.sink.hbase2;
+
+import java.util.List;
+
+import org.apache.flume.FlumeException;
+import org.apache.hadoop.hbase.client.Row;
+
+class MockSimpleHBase2EventSerializer extends SimpleHBase2EventSerializer {
+
+ public static boolean throwException = false;
+
+ @Override
+ public List<Row> getActions() throws FlumeException {
+ if (throwException) {
+ throw new FlumeException("Exception for testing");
+ }
+ return super.getActions();
+ }
+}
\ No newline at end of file
diff --git a/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/TestHBase2Sink.java b/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/TestHBase2Sink.java
new file mode 100644
index 0000000..21aa488
--- /dev/null
+++ b/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/TestHBase2Sink.java
@@ -0,0 +1,786 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hbase2;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.primitives.Longs;
+import org.apache.flume.Channel;
+import org.apache.flume.ChannelException;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.EventDeliveryException;
+import org.apache.flume.FlumeException;
+import org.apache.flume.Sink.Status;
+import org.apache.flume.Transaction;
+import org.apache.flume.channel.MemoryChannel;
+import org.apache.flume.conf.Configurables;
+import org.apache.flume.conf.ConfigurationException;
+import org.apache.flume.event.EventBuilder;
+import org.apache.flume.instrumentation.SinkCounter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKConfig;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.mockito.internal.util.reflection.Whitebox;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+
+import static org.mockito.Mockito.*;
+
+public class TestHBase2Sink {
+ private static final Logger logger =
+ LoggerFactory.getLogger(TestHBase2Sink.class);
+
+ private static final String tableName = "TestHbaseSink";
+ private static final String columnFamily = "TestColumnFamily";
+ private static final String inColumn = "iCol";
+ private static final String plCol = "pCol";
+ private static final String valBase = "testing hbase sink: jham";
+
+ private static HBaseTestingUtility testUtility;
+
+ private Configuration conf;
+
+ @BeforeClass
+ public static void setUpOnce() throws Exception {
+ String hbaseVer = org.apache.hadoop.hbase.util.VersionInfo.getVersion();
+ System.out.println("HBASE VERSION:" + hbaseVer);
+
+ Configuration conf = HBaseConfiguration.create();
+ conf.setBoolean("hbase.localcluster.assign.random.ports", true);
+ testUtility = new HBaseTestingUtility(conf);
+ testUtility.startMiniCluster();
+ }
+
+ @AfterClass
+ public static void tearDownOnce() throws Exception {
+ testUtility.shutdownMiniCluster();
+ }
+
+ /**
+ * Most common context setup for unit tests using
+ * {@link SimpleHBase2EventSerializer}.
+ */
+ @Before
+ public void setUp() throws IOException {
+ conf = new Configuration(testUtility.getConfiguration());
+ testUtility.createTable(TableName.valueOf(tableName), columnFamily.getBytes());
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ testUtility.deleteTable(TableName.valueOf(tableName));
+ }
+
+ /**
+ * Set up {@link Context} for use with {@link SimpleHBase2EventSerializer}.
+ */
+ private Context getContextForSimpleHBase2EventSerializer() {
+ Context ctx = new Context();
+ ctx.put("table", tableName);
+ ctx.put("columnFamily", columnFamily);
+ ctx.put("serializer", SimpleHBase2EventSerializer.class.getName());
+ ctx.put("serializer.payloadColumn", plCol);
+ ctx.put("serializer.incrementColumn", inColumn);
+ return ctx;
+ }
+
+ /**
+ * Set up {@link Context} for use with {@link IncrementHBase2Serializer}.
+ */
+ private Context getContextForIncrementHBaseSerializer() {
+ Context ctx = new Context();
+ ctx.put("table", tableName);
+ ctx.put("columnFamily", columnFamily);
+ ctx.put("serializer", IncrementHBase2Serializer.class.getName());
+ return ctx;
+ }
+
+ /**
+ * Set up {@link Context} for use with {@link IncrementHBase2Serializer}.
+ */
+ private Context getContextWithoutIncrementHBaseSerializer() {
+ //Create a context without setting increment column and payload Column
+ Context ctx = new Context();
+ ctx.put("table", tableName);
+ ctx.put("columnFamily", columnFamily);
+ ctx.put("serializer", SimpleHBase2EventSerializer.class.getName());
+ return ctx;
+ }
+
+ @Test
+ public void testOneEventWithDefaults() throws Exception {
+ Context ctx = getContextWithoutIncrementHBaseSerializer();
+ HBase2Sink sink = new HBase2Sink(conf);
+ Configurables.configure(sink, ctx);
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, new Context());
+ sink.setChannel(channel);
+ sink.start();
+
+ Transaction tx = channel.getTransaction();
+ tx.begin();
+ Event e = EventBuilder.withBody(Bytes.toBytes(valBase));
+ channel.put(e);
+ tx.commit();
+ tx.close();
+
+ sink.process();
+ sink.stop();
+
+ try (Connection connection = ConnectionFactory.createConnection(conf);
+ Table table = connection.getTable(TableName.valueOf(tableName))) {
+ byte[][] results = getResults(table, 1);
+ byte[] out = results[0];
+ Assert.assertArrayEquals(e.getBody(), out);
+ out = results[1];
+ Assert.assertArrayEquals(Longs.toByteArray(1), out);
+ }
+ }
+
+ @Test
+ public void testOneEvent() throws Exception {
+ Context ctx = getContextForSimpleHBase2EventSerializer();
+ HBase2Sink sink = new HBase2Sink(conf);
+ Configurables.configure(sink, ctx);
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, new Context());
+ sink.setChannel(channel);
+ sink.start();
+ Transaction tx = channel.getTransaction();
+ tx.begin();
+ Event e = EventBuilder.withBody(
+ Bytes.toBytes(valBase));
+ channel.put(e);
+ tx.commit();
+ tx.close();
+
+ sink.process();
+ sink.stop();
+
+ try (Connection connection = ConnectionFactory.createConnection(conf);
+ Table table = connection.getTable(TableName.valueOf(tableName))) {
+ byte[][] results = getResults(table, 1);
+ byte[] out = results[0];
+ Assert.assertArrayEquals(e.getBody(), out);
+ out = results[1];
+ Assert.assertArrayEquals(Longs.toByteArray(1), out);
+ }
+ }
+
+ @Test
+ public void testThreeEvents() throws Exception {
+ Context ctx = getContextForSimpleHBase2EventSerializer();
+ ctx.put("batchSize", "3");
+ HBase2Sink sink = new HBase2Sink(conf);
+ Configurables.configure(sink, ctx);
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, new Context());
+ sink.setChannel(channel);
+ sink.start();
+ Transaction tx = channel.getTransaction();
+ tx.begin();
+ for (int i = 0; i < 3; i++) {
+ Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
+ channel.put(e);
+ }
+ tx.commit();
+ tx.close();
+ sink.process();
+ sink.stop();
+
+ try (Connection connection = ConnectionFactory.createConnection(conf);
+ Table table = connection.getTable(TableName.valueOf(tableName))) {
+ byte[][] results = getResults(table, 3);
+ byte[] out;
+ int found = 0;
+ for (int i = 0; i < 3; i++) {
+ for (int j = 0; j < 3; j++) {
+ if (Arrays.equals(results[j], Bytes.toBytes(valBase + "-" + i))) {
+ found++;
+ break;
+ }
+ }
+ }
+ Assert.assertEquals(3, found);
+ out = results[3];
+ Assert.assertArrayEquals(Longs.toByteArray(3), out);
+ }
+ }
+
+ @Test
+ public void testMultipleBatches() throws Exception {
+ Context ctx = getContextForSimpleHBase2EventSerializer();
+ ctx.put("batchSize", "2");
+ HBase2Sink sink = new HBase2Sink(conf);
+ Configurables.configure(sink, ctx);
+ //Reset the context to a higher batchSize
+ ctx.put("batchSize", "100");
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, new Context());
+ sink.setChannel(channel);
+ sink.start();
+ Transaction tx = channel.getTransaction();
+ tx.begin();
+ for (int i = 0; i < 3; i++) {
+ Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
+ channel.put(e);
+ }
+ tx.commit();
+ tx.close();
+ int count = 0;
+ while (sink.process() != Status.BACKOFF) {
+ count++;
+ }
+ sink.stop();
+ Assert.assertEquals(2, count);
+
+ try (Connection connection = ConnectionFactory.createConnection(conf)) {
+ Table table = connection.getTable(TableName.valueOf(tableName));
+ byte[][] results = getResults(table, 3);
+ byte[] out;
+ int found = 0;
+ for (int i = 0; i < 3; i++) {
+ for (int j = 0; j < 3; j++) {
+ if (Arrays.equals(results[j], Bytes.toBytes(valBase + "-" + i))) {
+ found++;
+ break;
+ }
+ }
+ }
+ Assert.assertEquals(3, found);
+ out = results[3];
+ Assert.assertArrayEquals(Longs.toByteArray(3), out);
+ }
+ }
+
+ @Test(expected = FlumeException.class)
+ public void testMissingTable() throws Exception {
+ logger.info("Running testMissingTable()");
+ Context ctx = getContextForSimpleHBase2EventSerializer();
+
+ // setUp() will create the table, so we delete it.
+ logger.info("Deleting table {}", tableName);
+ testUtility.deleteTable(TableName.valueOf(tableName));
+
+ ctx.put("batchSize", "2");
+ HBase2Sink sink = new HBase2Sink(conf);
+ Configurables.configure(sink, ctx);
+ //Reset the context to a higher batchSize
+ ctx.put("batchSize", "100");
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, new Context());
+ sink.setChannel(channel);
+
+ logger.info("Writing data into channel");
+ Transaction tx = channel.getTransaction();
+ tx.begin();
+ for (int i = 0; i < 3; i++) {
+ Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
+ channel.put(e);
+ }
+ tx.commit();
+ tx.close();
+
+ logger.info("Starting sink and processing events");
+ try {
+ logger.info("Calling sink.start()");
+ sink.start(); // This method will throw.
+
+ // We never get here, but we log in case the behavior changes.
+ logger.error("Unexpected error: Calling sink.process()");
+ sink.process();
+ logger.error("Unexpected error: Calling sink.stop()");
+ sink.stop();
+ } finally {
+ // Re-create the table so tearDown() doesn't throw.
+ testUtility.createTable(TableName.valueOf(tableName), columnFamily.getBytes());
+ }
+
+ // FIXME: The test should never get here, the below code doesn't run.
+ Assert.fail();
+ }
+
+ // TODO: Move this test to a different class and run it stand-alone.
+
+ /**
+ * This test must run last - it shuts down the minicluster :D
+ *
+ * @throws Exception
+ */
+ @Ignore("For dev builds only:" +
+ "This test takes too long, and this has to be run after all other" +
+ "tests, since it shuts down the minicluster. " +
+ "Comment out all other tests" +
+ "and uncomment this annotation to run this test.")
+ @Test(expected = EventDeliveryException.class)
+ public void testHBaseFailure() throws Exception {
+ Context ctx = getContextForSimpleHBase2EventSerializer();
+ ctx.put("batchSize", "2");
+ HBase2Sink sink = new HBase2Sink(conf);
+ Configurables.configure(sink, ctx);
+ //Reset the context to a higher batchSize
+ ctx.put("batchSize", "100");
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, new Context());
+ sink.setChannel(channel);
+ sink.start();
+ Transaction tx = channel.getTransaction();
+ tx.begin();
+ for (int i = 0; i < 3; i++) {
+ Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
+ channel.put(e);
+ }
+ tx.commit();
+ tx.close();
+ sink.process();
+ try (Connection connection = ConnectionFactory.createConnection(conf);
+ Table table = connection.getTable(TableName.valueOf(tableName))) {
+ byte[][] results = getResults(table, 2);
+ byte[] out;
+ int found = 0;
+ for (int i = 0; i < 2; i++) {
+ for (int j = 0; j < 2; j++) {
+ if (Arrays.equals(results[j], Bytes.toBytes(valBase + "-" + i))) {
+ found++;
+ break;
+ }
+ }
+ }
+ Assert.assertEquals(2, found);
+ out = results[2];
+ Assert.assertArrayEquals(Longs.toByteArray(2), out);
+ }
+ testUtility.shutdownMiniCluster();
+ sink.process();
+ sink.stop();
+ }
+
+ /**
+ * Makes HBase scans to get rows in the payload column and increment column
+ * in the table given. Expensive, so tread lightly.
+ * Calling this function multiple times for the same result set is a bad
+ * idea. Cache the result set once it is returned by this function.
+ *
+ * @param table
+ * @param numEvents Number of events inserted into the table
+ * @return array of byte arrays
+ * @throws IOException
+ */
+ private byte[][] getResults(Table table, int numEvents) throws IOException {
+ byte[][] results = new byte[numEvents + 1][];
+ Scan scan = new Scan();
+ scan.addColumn(columnFamily.getBytes(),plCol.getBytes());
+ scan.withStartRow(Bytes.toBytes("default"));
+ ResultScanner rs = table.getScanner(scan);
+ byte[] out;
+ int i = 0;
+ try {
+ for (Result r = rs.next(); r != null; r = rs.next()) {
+ out = r.getValue(columnFamily.getBytes(), plCol.getBytes());
+
+ if (i >= results.length - 1) {
+ rs.close();
+ throw new FlumeException("More results than expected in the table." +
+ "Expected = " + numEvents + ". Found = " + i);
+ }
+ results[i++] = out;
+ System.out.println(out);
+ }
+ } finally {
+ rs.close();
+ }
+
+ Assert.assertEquals(i, results.length - 1);
+ scan = new Scan();
+ scan.addColumn(columnFamily.getBytes(),inColumn.getBytes());
+ scan.withStartRow(Bytes.toBytes("incRow"));
+ rs = table.getScanner(scan);
+ try {
+ for (Result r = rs.next(); r != null; r = rs.next()) {
+ out = r.getValue(columnFamily.getBytes(), inColumn.getBytes());
+ results[i++] = out;
+ System.out.println(out);
+ }
+ } finally {
+ rs.close();
+ }
+ return results;
+ }
+
+ @Test
+ public void testTransactionStateOnChannelException() throws Exception {
+ Context ctx = getContextForSimpleHBase2EventSerializer();
+ ctx.put("batchSize", "1");
+
+ HBase2Sink sink = new HBase2Sink(conf);
+ Configurables.configure(sink, ctx);
+ // Reset the context to a higher batchSize
+ Channel channel = spy(new MemoryChannel());
+ Configurables.configure(channel, new Context());
+ sink.setChannel(channel);
+ sink.start();
+ Transaction tx = channel.getTransaction();
+ tx.begin();
+ Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + 0));
+ channel.put(e);
+ tx.commit();
+ tx.close();
+ doThrow(new ChannelException("Mock Exception")).when(channel).take();
+ try {
+ sink.process();
+ Assert.fail("take() method should throw exception");
+ } catch (ChannelException ex) {
+ Assert.assertEquals("Mock Exception", ex.getMessage());
+ SinkCounter sinkCounter = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
+ Assert.assertEquals(1, sinkCounter.getChannelReadFail());
+ }
+ doReturn(e).when(channel).take();
+ sink.process();
+ sink.stop();
+
+ try (Connection connection = ConnectionFactory.createConnection(conf);
+ Table table = connection.getTable(TableName.valueOf(tableName))) {
+ byte[][] results = getResults(table, 1);
+ byte[] out = results[0];
+ Assert.assertArrayEquals(e.getBody(), out);
+ out = results[1];
+ Assert.assertArrayEquals(Longs.toByteArray(1), out);
+ }
+ }
+
+ @Test
+ public void testTransactionStateOnSerializationException() throws Exception {
+ Context ctx = getContextForSimpleHBase2EventSerializer();
+ ctx.put("batchSize", "1");
+ ctx.put(HBase2SinkConfigurationConstants.CONFIG_SERIALIZER,
+ "org.apache.flume.sink.hbase2.MockSimpleHBase2EventSerializer");
+
+ HBase2Sink sink = new HBase2Sink(conf);
+ Configurables.configure(sink, ctx);
+ // Reset the context to a higher batchSize
+ ctx.put("batchSize", "100");
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, new Context());
+ sink.setChannel(channel);
+ sink.start();
+ Transaction tx = channel.getTransaction();
+ tx.begin();
+ Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + 0));
+ channel.put(e);
+ tx.commit();
+ tx.close();
+ try {
+ MockSimpleHBase2EventSerializer.throwException = true;
+ sink.process();
+ Assert.fail("FlumeException expected from serializer");
+ } catch (FlumeException ex) {
+ Assert.assertEquals("Exception for testing", ex.getMessage());
+ SinkCounter sinkCounter = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
+ Assert.assertEquals(1, sinkCounter.getEventWriteFail());
+ }
+ MockSimpleHBase2EventSerializer.throwException = false;
+ sink.process();
+ sink.stop();
+
+ try (Connection connection = ConnectionFactory.createConnection(conf);
+ Table table = connection.getTable(TableName.valueOf(tableName))) {
+ byte[][] results = getResults(table, 1);
+ byte[] out = results[0];
+ Assert.assertArrayEquals(e.getBody(), out);
+ out = results[1];
+ Assert.assertArrayEquals(Longs.toByteArray(1), out);
+ }
+ }
+
+ @Test
+ public void testWithoutConfigurationObject() throws Exception {
+ Context ctx = getContextForSimpleHBase2EventSerializer();
+ Context tmpContext = new Context(ctx.getParameters());
+ tmpContext.put("batchSize", "2");
+ tmpContext.put(HBase2SinkConfigurationConstants.ZK_QUORUM,
+ ZKConfig.getZKQuorumServersString(conf));
+ System.out.print(ctx.getString(HBase2SinkConfigurationConstants.ZK_QUORUM));
+ tmpContext.put(HBase2SinkConfigurationConstants.ZK_ZNODE_PARENT,
+ conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
+ HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT));
+
+ HBase2Sink sink = new HBase2Sink();
+ Configurables.configure(sink, tmpContext);
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, ctx);
+ sink.setChannel(channel);
+ sink.start();
+ Transaction tx = channel.getTransaction();
+ tx.begin();
+ for (int i = 0; i < 3; i++) {
+ Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
+ channel.put(e);
+ }
+ tx.commit();
+ tx.close();
+ Status status = Status.READY;
+ while (status != Status.BACKOFF) {
+ status = sink.process();
+ }
+ sink.stop();
+
+ try (Connection connection = ConnectionFactory.createConnection(conf);
+ Table table = connection.getTable(TableName.valueOf(tableName))) {
+ byte[][] results = getResults(table, 3);
+ byte[] out;
+ int found = 0;
+ for (int i = 0; i < 3; i++) {
+ for (int j = 0; j < 3; j++) {
+ if (Arrays.equals(results[j], Bytes.toBytes(valBase + "-" + i))) {
+ found++;
+ break;
+ }
+ }
+ }
+ Assert.assertEquals(3, found);
+ out = results[3];
+ Assert.assertArrayEquals(Longs.toByteArray(3), out);
+ }
+ }
+
+ @Test
+ public void testZKQuorum() throws Exception {
+ Context ctx = getContextForSimpleHBase2EventSerializer();
+ Context tmpContext = new Context(ctx.getParameters());
+ String zkQuorum = "zk1.flume.apache.org:3342, zk2.flume.apache.org:3342, " +
+ "zk3.flume.apache.org:3342";
+ tmpContext.put("batchSize", "2");
+ tmpContext.put(HBase2SinkConfigurationConstants.ZK_QUORUM, zkQuorum);
+ tmpContext.put(HBase2SinkConfigurationConstants.ZK_ZNODE_PARENT,
+ conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
+ HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT));
+ HBase2Sink sink = new HBase2Sink();
+ Configurables.configure(sink, tmpContext);
+ Assert.assertEquals("zk1.flume.apache.org,zk2.flume.apache.org," +
+ "zk3.flume.apache.org", sink.getConfig().get(HConstants.ZOOKEEPER_QUORUM));
+ Assert.assertEquals(String.valueOf(3342),
+ sink.getConfig().get(HConstants.ZOOKEEPER_CLIENT_PORT));
+ }
+
+ @Test(expected = FlumeException.class)
+ public void testZKQuorumIncorrectPorts() throws Exception {
+ Context ctx = getContextForSimpleHBase2EventSerializer();
+ Context tmpContext = new Context(ctx.getParameters());
+
+ String zkQuorum = "zk1.flume.apache.org:3345, zk2.flume.apache.org:3342, " +
+ "zk3.flume.apache.org:3342";
+ tmpContext.put("batchSize", "2");
+ tmpContext.put(HBase2SinkConfigurationConstants.ZK_QUORUM, zkQuorum);
+ tmpContext.put(HBase2SinkConfigurationConstants.ZK_ZNODE_PARENT,
+ conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
+ HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT));
+ HBase2Sink sink = new HBase2Sink();
+ Configurables.configure(sink, tmpContext);
+ Assert.fail();
+ }
+
+ @Test
+ public void testCoalesce() throws EventDeliveryException {
+ Context ctx = getContextForIncrementHBaseSerializer();
+ ctx.put("batchSize", "100");
+ ctx.put(HBase2SinkConfigurationConstants.CONFIG_COALESCE_INCREMENTS,
+ String.valueOf(true));
+
+ final Map<String, Long> expectedCounts = Maps.newHashMap();
+ expectedCounts.put("r1:c1", 10L);
+ expectedCounts.put("r1:c2", 20L);
+ expectedCounts.put("r2:c1", 7L);
+ expectedCounts.put("r2:c3", 63L);
+ HBase2Sink.DebugIncrementsCallback cb = new CoalesceValidator(expectedCounts);
+
+ HBase2Sink sink = new HBase2Sink(testUtility.getConfiguration(), cb);
+ Configurables.configure(sink, ctx);
+ Channel channel = createAndConfigureMemoryChannel(sink);
+
+ List<Event> events = Lists.newLinkedList();
+ generateEvents(events, expectedCounts);
+ putEvents(channel, events);
+
+ sink.start();
+ sink.process(); // Calls CoalesceValidator instance.
+ sink.stop();
+ }
+
+ @Test(expected = AssertionError.class)
+ public void negativeTestCoalesce() throws EventDeliveryException {
+ Context ctx = getContextForIncrementHBaseSerializer();
+ ctx.put("batchSize", "10");
+
+ final Map<String, Long> expectedCounts = Maps.newHashMap();
+ expectedCounts.put("r1:c1", 10L);
+ HBase2Sink.DebugIncrementsCallback cb = new CoalesceValidator(expectedCounts);
+
+ HBase2Sink sink = new HBase2Sink(testUtility.getConfiguration(), cb);
+ Configurables.configure(sink, ctx);
+ Channel channel = createAndConfigureMemoryChannel(sink);
+
+ List<Event> events = Lists.newLinkedList();
+ generateEvents(events, expectedCounts);
+ putEvents(channel, events);
+
+ sink.start();
+ sink.process(); // Calls CoalesceValidator instance.
+ sink.stop();
+ }
+
+ @Test
+ public void testBatchAware() throws EventDeliveryException {
+ logger.info("Running testBatchAware()");
+ Context ctx = getContextForIncrementHBaseSerializer();
+ HBase2Sink sink = new HBase2Sink(testUtility.getConfiguration());
+ Configurables.configure(sink, ctx);
+ Channel channel = createAndConfigureMemoryChannel(sink);
+
+ sink.start();
+ int batchCount = 3;
+ for (int i = 0; i < batchCount; i++) {
+ sink.process();
+ }
+ sink.stop();
+ Assert.assertEquals(batchCount,
+ ((IncrementHBase2Serializer) sink.getSerializer()).getNumBatchesStarted());
+ }
+
+ @Test (expected = ConfigurationException.class)
+ public void testHBaseVersionCheck() throws Exception {
+ Context ctx = getContextWithoutIncrementHBaseSerializer();
+ HBase2Sink sink = mock(HBase2Sink.class);
+ doCallRealMethod().when(sink).configure(any());
+ when(sink.getHBbaseVersionString()).thenReturn("1.0.0");
+ Configurables.configure(sink, ctx);
+ }
+
+ @Test (expected = ConfigurationException.class)
+ public void testHBaseVersionCheckNotANumber() throws Exception {
+ Context ctx = getContextWithoutIncrementHBaseSerializer();
+ HBase2Sink sink = mock(HBase2Sink.class);
+ doCallRealMethod().when(sink).configure(any());
+ when(sink.getHBbaseVersionString()).thenReturn("Dummy text");
+ Configurables.configure(sink, ctx);
+ }
+
+ /**
+ * For testing that the rows coalesced, serialized by
+ * {@link IncrementHBase2Serializer}, are of the expected batch size.
+ */
+ private static class CoalesceValidator
+ implements HBase2Sink.DebugIncrementsCallback {
+
+ private final Map<String,Long> expectedCounts;
+
+ public CoalesceValidator(Map<String, Long> expectedCounts) {
+ this.expectedCounts = expectedCounts;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void onAfterCoalesce(Iterable<Increment> increments) {
+ for (Increment inc : increments) {
+ byte[] row = inc.getRow();
+ Map<byte[], NavigableMap<byte[], Long>> families = null;
+ try {
+ families = inc.getFamilyMapOfLongs();
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ }
+ assert families != null;
+ for (byte[] family : families.keySet()) {
+ NavigableMap<byte[], Long> qualifiers = families.get(family);
+ for (Map.Entry<byte[], Long> entry : qualifiers.entrySet()) {
+ byte[] qualifier = entry.getKey();
+ Long count = entry.getValue();
+ String key = new String(row, Charsets.UTF_8) +
+ ':' +
+ new String(qualifier, Charsets.UTF_8);
+ Assert.assertEquals("Expected counts don't match observed for " + key,
+ expectedCounts.get(key), count);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Add number of Events corresponding to counts to the events list.
+ * @param events Destination list.
+ * @param counts How many events to generate for each row:qualifier pair.
+ */
+ private void generateEvents(List<Event> events, Map<String, Long> counts) {
+ for (String key : counts.keySet()) {
+ long count = counts.get(key);
+ for (long i = 0; i < count; i++) {
+ events.add(EventBuilder.withBody(key, Charsets.UTF_8));
+ }
+ }
+ }
+
+ private Channel createAndConfigureMemoryChannel(HBase2Sink sink) {
+ Channel channel = new MemoryChannel();
+ Context channelCtx = new Context();
+ channelCtx.put("capacity", String.valueOf(1000L));
+ channelCtx.put("transactionCapacity", String.valueOf(1000L));
+ Configurables.configure(channel, channelCtx);
+ sink.setChannel(channel);
+ channel.start();
+ return channel;
+ }
+
+ private void putEvents(Channel channel, Iterable<Event> events) {
+ Transaction tx = channel.getTransaction();
+ tx.begin();
+ for (Event event : events) {
+ channel.put(event);
+ }
+ tx.commit();
+ tx.close();
+ }
+
+}
diff --git a/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/TestHBase2SinkCreation.java b/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/TestHBase2SinkCreation.java
new file mode 100644
index 0000000..02127ba
--- /dev/null
+++ b/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/TestHBase2SinkCreation.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hbase2;
+
+import org.apache.flume.FlumeException;
+import org.apache.flume.Sink;
+import org.apache.flume.SinkFactory;
+import org.apache.flume.sink.DefaultSinkFactory;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestHBase2SinkCreation {
+
+ private SinkFactory sinkFactory;
+
+ @Before
+ public void setUp() {
+ sinkFactory = new DefaultSinkFactory();
+ }
+
+ private void verifySinkCreation(Class<?> typeClass) throws FlumeException {
+ Sink sink = sinkFactory.create("hbase2-sink", "hbase2");
+ Assert.assertNotNull(sink);
+ Assert.assertTrue(typeClass.isInstance(sink));
+ }
+
+ @Test
+ public void testSinkCreation() {
+ verifySinkCreation(HBase2Sink.class);
+ }
+}
diff --git a/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/TestRegexHBase2EventSerializer.java b/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/TestRegexHBase2EventSerializer.java
new file mode 100644
index 0000000..ee9f739
--- /dev/null
+++ b/flume-hbase2-sink/src/test/java/org/apache/flume/sink/hbase2/TestRegexHBase2EventSerializer.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hbase2;
+
+import com.google.common.collect.Maps;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.event.EventBuilder;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+
+import java.nio.charset.Charset;
+import java.util.Calendar;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestRegexHBase2EventSerializer {
+
+ @Test
+ /* Ensure that when no config is specified, the a catch-all regex is used
+ * with default column name. */
+ public void testDefaultBehavior() throws Exception {
+ RegexHBase2EventSerializer s = new RegexHBase2EventSerializer();
+ Context context = new Context();
+ s.configure(context);
+ String logMsg = "The sky is falling!";
+ Event e = EventBuilder.withBody(Bytes.toBytes(logMsg));
+ s.initialize(e, "CF".getBytes());
+ List<Row> actions = s.getActions();
+ assertTrue(actions.size() == 1);
+ assertTrue(actions.get(0) instanceof Put);
+ Put put = (Put) actions.get(0);
+
+ assertTrue(put.getFamilyCellMap().containsKey(s.cf));
+ List<Cell> cells = put.getFamilyCellMap().get(s.cf);
+ assertTrue(cells.size() == 1);
+
+ Map<String, String> resultMap = Maps.newHashMap();
+ for (Cell cell : cells) {
+ resultMap.put(new String(CellUtil.cloneQualifier(cell)),
+ new String(CellUtil.cloneValue(cell)));
+ }
+
+ assertTrue(resultMap.containsKey(
+ RegexHBase2EventSerializer.COLUMN_NAME_DEFAULT));
+ assertEquals("The sky is falling!",
+ resultMap.get(RegexHBase2EventSerializer.COLUMN_NAME_DEFAULT));
+ }
+ @Test
+ public void testRowIndexKey() throws Exception {
+ RegexHBase2EventSerializer s = new RegexHBase2EventSerializer();
+ Context context = new Context();
+ context.put(RegexHBase2EventSerializer.REGEX_CONFIG,"^([^\t]+)\t([^\t]+)\t" + "([^\t]+)$");
+ context.put(RegexHBase2EventSerializer.COL_NAME_CONFIG, "col1,col2,ROW_KEY");
+ context.put("rowKeyIndex", "2");
+ s.configure(context);
+
+ String body = "val1\tval2\trow1";
+ Event e = EventBuilder.withBody(Bytes.toBytes(body));
+ s.initialize(e, "CF".getBytes());
+ List<Row> actions = s.getActions();
+
+ Put put = (Put)actions.get(0);
+
+ List<Cell> cells = put.getFamilyCellMap().get(s.cf);
+ assertTrue(cells.size() == 2);
+
+ Map<String, String> resultMap = Maps.newHashMap();
+ for (Cell cell : cells) {
+ resultMap.put(new String(CellUtil.cloneQualifier(cell)),
+ new String(CellUtil.cloneValue(cell)));
+ }
+ assertEquals("val1", resultMap.get("col1"));
+ assertEquals("val2", resultMap.get("col2"));
+ assertEquals("row1", Bytes.toString(put.getRow()));
+ }
+
+ @Test
+ /* Test a common case where regex is used to parse apache log format. */
+ public void testApacheRegex() throws Exception {
+ RegexHBase2EventSerializer s = new RegexHBase2EventSerializer();
+ Context context = new Context();
+ context.put(RegexHBase2EventSerializer.REGEX_CONFIG,
+ "([^ ]*) ([^ ]*) ([^ ]*) (-|\\[[^\\]]*\\]) \"([^ ]+) ([^ ]+)" +
+ " ([^\"]+)\" (-|[0-9]*) (-|[0-9]*)(?: ([^ \"]*|\"[^\"]*\")" +
+ " ([^ \"]*|\"[^\"]*\"))?");
+ context.put(RegexHBase2EventSerializer.COL_NAME_CONFIG,
+ "host,identity,user,time,method,request,protocol,status,size," +
+ "referer,agent");
+ s.configure(context);
+ String logMsg = "33.22.11.00 - - [20/May/2011:07:01:19 +0000] " +
+ "\"GET /wp-admin/css/install.css HTTP/1.0\" 200 813 " +
+ "\"http://www.cloudera.com/wp-admin/install.php\" \"Mozilla/5.0 (comp" +
+ "atible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)\"";
+
+ Event e = EventBuilder.withBody(Bytes.toBytes(logMsg));
+ s.initialize(e, "CF".getBytes());
+ List<Row> actions = s.getActions();
+ assertEquals(1, s.getActions().size());
+ assertTrue(actions.get(0) instanceof Put);
+
+ Put put = (Put) actions.get(0);
+ assertTrue(put.getFamilyCellMap().containsKey(s.cf));
+ List<Cell> cells = put.getFamilyCellMap().get(s.cf);
+ assertTrue(cells.size() == 11);
+
+ Map<String, String> resultMap = Maps.newHashMap();
+ for (Cell cell : cells) {
+ resultMap.put(new String(CellUtil.cloneQualifier(cell)),
+ new String(CellUtil.cloneValue(cell)));
+ }
+
+ assertEquals("33.22.11.00", resultMap.get("host"));
+ assertEquals("-", resultMap.get("identity"));
+ assertEquals("-", resultMap.get("user"));
+ assertEquals("[20/May/2011:07:01:19 +0000]", resultMap.get("time"));
+ assertEquals("GET", resultMap.get("method"));
+ assertEquals("/wp-admin/css/install.css", resultMap.get("request"));
+ assertEquals("HTTP/1.0", resultMap.get("protocol"));
+ assertEquals("200", resultMap.get("status"));
+ assertEquals("813", resultMap.get("size"));
+ assertEquals("\"http://www.cloudera.com/wp-admin/install.php\"",
+ resultMap.get("referer"));
+ assertEquals("\"Mozilla/5.0 (compatible; Yahoo! Slurp; " +
+ "http://help.yahoo.com/help/us/ysearch/slurp)\"",
+ resultMap.get("agent"));
+
+ List<Increment> increments = s.getIncrements();
+ assertEquals(0, increments.size());
+ }
+
+ @Test
+ public void testRowKeyGeneration() {
+ Context context = new Context();
+ RegexHBase2EventSerializer s1 = new RegexHBase2EventSerializer();
+ s1.configure(context);
+ RegexHBase2EventSerializer s2 = new RegexHBase2EventSerializer();
+ s2.configure(context);
+
+ // Reset shared nonce to zero
+ RegexHBase2EventSerializer.nonce.set(0);
+ String randomString = RegexHBase2EventSerializer.randomKey;
+
+ Event e1 = EventBuilder.withBody(Bytes.toBytes("body"));
+ Event e2 = EventBuilder.withBody(Bytes.toBytes("body"));
+ Event e3 = EventBuilder.withBody(Bytes.toBytes("body"));
+
+ Calendar cal = mock(Calendar.class);
+ when(cal.getTimeInMillis()).thenReturn(1L);
+
+ s1.initialize(e1, "CF".getBytes());
+ String rk1 = new String(s1.getRowKey(cal));
+ assertEquals("1-" + randomString + "-0", rk1);
+
+ when(cal.getTimeInMillis()).thenReturn(10L);
+ s1.initialize(e2, "CF".getBytes());
+ String rk2 = new String(s1.getRowKey(cal));
+ assertEquals("10-" + randomString + "-1", rk2);
+
+ when(cal.getTimeInMillis()).thenReturn(100L);
+ s2.initialize(e3, "CF".getBytes());
+ String rk3 = new String(s2.getRowKey(cal));
+ assertEquals("100-" + randomString + "-2", rk3);
+
+ }
+
+ @Test
+ /* Test depositing of the header information. */
+ public void testDepositHeaders() throws Exception {
+ Charset charset = Charset.forName("KOI8-R");
+ RegexHBase2EventSerializer s = new RegexHBase2EventSerializer();
+ Context context = new Context();
+ context.put(RegexHBase2EventSerializer.DEPOSIT_HEADERS_CONFIG,
+ "true");
+ context.put(RegexHBase2EventSerializer.CHARSET_CONFIG,
+ charset.toString());
+ s.configure(context);
+
+ String body = "body";
+ Map<String, String> headers = Maps.newHashMap();
+ headers.put("header1", "value1");
+ headers.put("заголовок2", "значение2");
+
+ Event e = EventBuilder.withBody(Bytes.toBytes(body), headers);
+ s.initialize(e, "CF".getBytes());
+ List<Row> actions = s.getActions();
+ assertEquals(1, s.getActions().size());
+ assertTrue(actions.get(0) instanceof Put);
+
+ Put put = (Put) actions.get(0);
+ assertTrue(put.getFamilyCellMap().containsKey(s.cf));
+ List<Cell> cells = put.getFamilyCellMap().get(s.cf);
+ assertTrue(cells.size() == 3);
+
+ Map<String, byte[]> resultMap = Maps.newHashMap();
+ for (Cell cell : cells) {
+ resultMap.put(new String(CellUtil.cloneQualifier(cell), charset),
+ CellUtil.cloneValue(cell));
+ }
+
+ assertEquals(body,
+ new String(resultMap.get(RegexHBase2EventSerializer.COLUMN_NAME_DEFAULT),
+ charset));
+ assertEquals("value1", new String(resultMap.get("header1"), charset));
+ assertArrayEquals("значение2".getBytes(charset), resultMap.get("заголовок2"));
+ assertEquals("значение2".length(), resultMap.get("заголовок2").length);
+
+ List<Increment> increments = s.getIncrements();
+ assertEquals(0, increments.size());
+ }
+}
diff --git a/flume-hdfs-sink/pom.xml b/flume-hdfs-sink/pom.xml
new file mode 100644
index 0000000..0038fb9
--- /dev/null
+++ b/flume-hdfs-sink/pom.xml
@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>flume-hadoop-parent</artifactId>
+ <groupId>org.apache.flume</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-hdfs-sink</artifactId>
+ <name>Flume HDFS Sink</name>
+
+ <properties>
+ <!-- TODO fix spotbugs/pmd violations -->
+ <spotbugs.maxAllowedViolations>14</spotbugs.maxAllowedViolations>
+ <pmd.maxAllowedViolations>25</pmd.maxAllowedViolations>
+ <module.name>org.apache.flume.sink.hdfs</module.name>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-sdk</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-configuration</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+
+ </dependencies>
+
+</project>
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AbstractHDFSWriter.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AbstractHDFSWriter.java
new file mode 100644
index 0000000..d2a07fa
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AbstractHDFSWriter.java
@@ -0,0 +1,281 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flume.sink.hdfs;
+
+import com.google.common.base.Preconditions;
+import org.apache.flume.Context;
+import org.apache.flume.FlumeException;
+import org.apache.flume.annotations.InterfaceAudience;
+import org.apache.flume.annotations.InterfaceStability;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class AbstractHDFSWriter implements HDFSWriter {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(AbstractHDFSWriter.class);
+
+ private FSDataOutputStream outputStream;
+ private FileSystem fs;
+ private Path destPath;
+ private Method refGetNumCurrentReplicas = null;
+ private Method refGetDefaultReplication = null;
+ private Method refHflushOrSync = null;
+ private Integer configuredMinReplicas = null;
+ private Integer numberOfCloseRetries = null;
+ private long timeBetweenCloseRetries = Long.MAX_VALUE;
+
+ static final Object[] NO_ARGS = new Object[]{};
+
+ @Override
+ public void configure(Context context) {
+ configuredMinReplicas = context.getInteger("hdfs.minBlockReplicas");
+ if (configuredMinReplicas != null) {
+ Preconditions.checkArgument(configuredMinReplicas >= 0,
+ "hdfs.minBlockReplicas must be greater than or equal to 0");
+ }
+ numberOfCloseRetries = context.getInteger("hdfs.closeTries", 1) - 1;
+
+ if (numberOfCloseRetries > 1) {
+ try {
+ //hdfs.callTimeout is deprecated from 1.9
+ timeBetweenCloseRetries = context.getLong("hdfs.callTimeout", 30000L);
+ } catch (NumberFormatException e) {
+ logger.warn("hdfs.callTimeout can not be parsed to a long: " +
+ context.getLong("hdfs.callTimeout"));
+ }
+ timeBetweenCloseRetries = Math.max(timeBetweenCloseRetries / numberOfCloseRetries, 1000);
+ }
+
+ }
+
+ /**
+ * Contract for subclasses: Call registerCurrentStream() on open,
+ * unregisterCurrentStream() on close, and the base class takes care of the
+ * rest.
+ * @return
+ */
+ @Override
+ public boolean isUnderReplicated() {
+ try {
+ int numBlocks = getNumCurrentReplicas();
+ if (numBlocks == -1) {
+ return false;
+ }
+ int desiredBlocks;
+ if (configuredMinReplicas != null) {
+ desiredBlocks = configuredMinReplicas;
+ } else {
+ desiredBlocks = getFsDesiredReplication();
+ }
+ return numBlocks < desiredBlocks;
+ } catch (IllegalAccessException e) {
+ logger.error("Unexpected error while checking replication factor", e);
+ } catch (InvocationTargetException e) {
+ logger.error("Unexpected error while checking replication factor", e);
+ } catch (IllegalArgumentException e) {
+ logger.error("Unexpected error while checking replication factor", e);
+ }
+ return false;
+ }
+
+ protected void registerCurrentStream(FSDataOutputStream outputStream,
+ FileSystem fs, Path destPath) {
+ Preconditions.checkNotNull(outputStream, "outputStream must not be null");
+ Preconditions.checkNotNull(fs, "fs must not be null");
+ Preconditions.checkNotNull(destPath, "destPath must not be null");
+
+ this.outputStream = outputStream;
+ this.fs = fs;
+ this.destPath = destPath;
+ this.refGetNumCurrentReplicas = reflectGetNumCurrentReplicas(outputStream);
+ this.refGetDefaultReplication = reflectGetDefaultReplication(fs);
+ this.refHflushOrSync = reflectHflushOrSync(outputStream);
+
+ }
+
+ protected void unregisterCurrentStream() {
+ this.outputStream = null;
+ this.fs = null;
+ this.destPath = null;
+ this.refGetNumCurrentReplicas = null;
+ this.refGetDefaultReplication = null;
+ }
+
+ public int getFsDesiredReplication() {
+ short replication = 0;
+ if (fs != null && destPath != null) {
+ if (refGetDefaultReplication != null) {
+ try {
+ replication = (Short) refGetDefaultReplication.invoke(fs, destPath);
+ } catch (IllegalAccessException e) {
+ logger.warn("Unexpected error calling getDefaultReplication(Path)", e);
+ } catch (InvocationTargetException e) {
+ logger.warn("Unexpected error calling getDefaultReplication(Path)", e);
+ }
+ } else {
+ // will not work on Federated HDFS (see HADOOP-8014)
+ replication = fs.getDefaultReplication();
+ }
+ }
+ return replication;
+ }
+
+ /**
+ * This method gets the datanode replication count for the current open file.
+ *
+ * If the pipeline isn't started yet or is empty, you will get the default
+ * replication factor.
+ *
+ * <p/>If this function returns -1, it means you
+ * are not properly running with the HDFS-826 patch.
+ * @throws InvocationTargetException
+ * @throws IllegalAccessException
+ * @throws IllegalArgumentException
+ */
+ public int getNumCurrentReplicas()
+ throws IllegalArgumentException, IllegalAccessException,
+ InvocationTargetException {
+ if (refGetNumCurrentReplicas != null && outputStream != null) {
+ OutputStream dfsOutputStream = outputStream.getWrappedStream();
+ if (dfsOutputStream != null) {
+ Object repl = refGetNumCurrentReplicas.invoke(dfsOutputStream, NO_ARGS);
+ if (repl instanceof Integer) {
+ return ((Integer)repl).intValue();
+ }
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Find the 'getNumCurrentReplicas' on the passed <code>os</code> stream.
+ * @return Method or null.
+ */
+ private Method reflectGetNumCurrentReplicas(FSDataOutputStream os) {
+ Method m = null;
+ if (os != null) {
+ Class<? extends OutputStream> wrappedStreamClass = os.getWrappedStream()
+ .getClass();
+ try {
+ m = wrappedStreamClass.getDeclaredMethod("getNumCurrentReplicas",
+ new Class<?>[] {});
+ m.setAccessible(true);
+ } catch (NoSuchMethodException e) {
+ logger.info("FileSystem's output stream doesn't support"
+ + " getNumCurrentReplicas; --HDFS-826 not available; fsOut="
+ + wrappedStreamClass.getName() + "; err=" + e);
+ } catch (SecurityException e) {
+ logger.info("Doesn't have access to getNumCurrentReplicas on "
+ + "FileSystems's output stream --HDFS-826 not available; fsOut="
+ + wrappedStreamClass.getName(), e);
+ m = null; // could happen on setAccessible()
+ }
+ }
+ if (m != null) {
+ logger.debug("Using getNumCurrentReplicas--HDFS-826");
+ }
+ return m;
+ }
+
+ /**
+ * Find the 'getDefaultReplication' method on the passed <code>fs</code>
+ * FileSystem that takes a Path argument.
+ * @return Method or null.
+ */
+ private Method reflectGetDefaultReplication(FileSystem fileSystem) {
+ Method m = null;
+ if (fileSystem != null) {
+ Class<?> fsClass = fileSystem.getClass();
+ try {
+ m = fsClass.getMethod("getDefaultReplication",
+ new Class<?>[] { Path.class });
+ } catch (NoSuchMethodException e) {
+ logger.debug("FileSystem implementation doesn't support"
+ + " getDefaultReplication(Path); -- HADOOP-8014 not available; " +
+ "className = " + fsClass.getName() + "; err = " + e);
+ } catch (SecurityException e) {
+ logger.debug("No access to getDefaultReplication(Path) on "
+ + "FileSystem implementation -- HADOOP-8014 not available; " +
+ "className = " + fsClass.getName() + "; err = " + e);
+ }
+ }
+ if (m != null) {
+ logger.debug("Using FileSystem.getDefaultReplication(Path) from " +
+ "HADOOP-8014");
+ }
+ return m;
+ }
+
+ private Method reflectHflushOrSync(FSDataOutputStream os) {
+ Method m = null;
+ if (os != null) {
+ Class<?> fsDataOutputStreamClass = os.getClass();
+ try {
+ m = fsDataOutputStreamClass.getMethod("hflush");
+ } catch (NoSuchMethodException ex) {
+ logger.debug("HFlush not found. Will use sync() instead");
+ try {
+ m = fsDataOutputStreamClass.getMethod("sync");
+ } catch (Exception ex1) {
+ String msg = "Neither hflush not sync were found. That seems to be " +
+ "a problem!";
+ logger.error(msg);
+ throw new FlumeException(msg, ex1);
+ }
+ }
+ }
+ return m;
+ }
+
+ /**
+ * If hflush is available in this version of HDFS, then this method calls
+ * hflush, else it calls sync.
+ * @param os - The stream to flush/sync
+ * @throws IOException
+ */
+ protected void hflushOrSync(FSDataOutputStream os) throws IOException {
+ try {
+ // At this point the refHflushOrSync cannot be null,
+ // since register method would have thrown if it was.
+ this.refHflushOrSync.invoke(os);
+ } catch (InvocationTargetException e) {
+ String msg = "Error while trying to hflushOrSync!";
+ logger.error(msg);
+ Throwable cause = e.getCause();
+ if (cause != null && cause instanceof IOException) {
+ throw (IOException)cause;
+ }
+ throw new FlumeException(msg, e);
+ } catch (Exception e) {
+ String msg = "Error while trying to hflushOrSync!";
+ logger.error(msg);
+ throw new FlumeException(msg, e);
+ }
+ }
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AvroEventSerializer.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AvroEventSerializer.java
new file mode 100644
index 0000000..3231742
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AvroEventSerializer.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hdfs;
+
+import org.apache.avro.AvroRuntimeException;
+import org.apache.avro.Schema;
+import org.apache.avro.file.CodecFactory;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.io.DatumWriter;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.FlumeException;
+import org.apache.flume.conf.Configurable;
+import org.apache.flume.serialization.EventSerializer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.COMPRESSION_CODEC;
+import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_COMPRESSION_CODEC;
+import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_STATIC_SCHEMA_URL;
+import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.DEFAULT_SYNC_INTERVAL_BYTES;
+import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.STATIC_SCHEMA_URL;
+import static org.apache.flume.serialization.AvroEventSerializerConfigurationConstants.SYNC_INTERVAL_BYTES;
+
+/**
+ * <p>
+ * This class serializes Flume {@linkplain org.apache.flume.Event events} into Avro data files. The
+ * Flume event body is read as an Avro datum, and is then written to the
+ * {@link org.apache.flume.serialization.EventSerializer}'s output stream in Avro data file format.
+ * </p>
+ * <p>
+ * The Avro schema is determined by reading a Flume event header. The schema may be
+ * specified either as a literal, by setting {@link #AVRO_SCHEMA_LITERAL_HEADER} (not
+ * recommended, since the full schema must be transmitted in every event),
+ * or as a URL which the schema may be read from, by setting {@link
+ * #AVRO_SCHEMA_URL_HEADER}. Schemas read from URLs are cached by instances of this
+ * class so that the overhead of retrieval is minimized.
+ * </p>
+ */
+public class AvroEventSerializer implements EventSerializer, Configurable {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(AvroEventSerializer.class);
+
+ public static final String AVRO_SCHEMA_LITERAL_HEADER = "flume.avro.schema.literal";
+ public static final String AVRO_SCHEMA_URL_HEADER = "flume.avro.schema.url";
+
+ private final OutputStream out;
+ private DatumWriter<Object> writer = null;
+ private DataFileWriter<Object> dataFileWriter = null;
+
+ private int syncIntervalBytes;
+ private String compressionCodec;
+ private Map<String, Schema> schemaCache = new HashMap<String, Schema>();
+ private String staticSchemaURL;
+
+ private AvroEventSerializer(OutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public void configure(Context context) {
+ syncIntervalBytes =
+ context.getInteger(SYNC_INTERVAL_BYTES, DEFAULT_SYNC_INTERVAL_BYTES);
+ compressionCodec =
+ context.getString(COMPRESSION_CODEC, DEFAULT_COMPRESSION_CODEC);
+ staticSchemaURL = context.getString(STATIC_SCHEMA_URL, DEFAULT_STATIC_SCHEMA_URL);
+ }
+
+ @Override
+ public void afterCreate() throws IOException {
+ // no-op
+ }
+
+ @Override
+ public void afterReopen() throws IOException {
+ // impossible to initialize DataFileWriter without writing the schema?
+ throw new UnsupportedOperationException("Avro API doesn't support append");
+ }
+
+ @Override
+ public void write(Event event) throws IOException {
+ if (dataFileWriter == null) {
+ initialize(event);
+ }
+ dataFileWriter.appendEncoded(ByteBuffer.wrap(event.getBody()));
+ }
+
+ private void initialize(Event event) throws IOException {
+ Schema schema = null;
+ String schemaUrl = event.getHeaders().get(AVRO_SCHEMA_URL_HEADER);
+ String schemaString = event.getHeaders().get(AVRO_SCHEMA_LITERAL_HEADER);
+
+ if (schemaUrl != null) { // if URL_HEADER is there then use it
+ schema = schemaCache.get(schemaUrl);
+ if (schema == null) {
+ schema = loadFromUrl(schemaUrl);
+ schemaCache.put(schemaUrl, schema);
+ }
+ } else if (schemaString != null) { // fallback to LITERAL_HEADER if it was there
+ schema = new Schema.Parser().parse(schemaString);
+ } else if (staticSchemaURL != null) { // fallback to static url if it was there
+ schema = schemaCache.get(staticSchemaURL);
+ if (schema == null) {
+ schema = loadFromUrl(staticSchemaURL);
+ schemaCache.put(staticSchemaURL, schema);
+ }
+ } else { // no other options so giving up
+ throw new FlumeException("Could not find schema for event " + event);
+ }
+
+ writer = new GenericDatumWriter<Object>(schema);
+ dataFileWriter = new DataFileWriter<Object>(writer);
+
+ dataFileWriter.setSyncInterval(syncIntervalBytes);
+
+ try {
+ CodecFactory codecFactory = CodecFactory.fromString(compressionCodec);
+ dataFileWriter.setCodec(codecFactory);
+ } catch (AvroRuntimeException e) {
+ logger.warn("Unable to instantiate avro codec with name (" +
+ compressionCodec + "). Compression disabled. Exception follows.", e);
+ }
+
+ dataFileWriter.create(schema, out);
+ }
+
+ private Schema loadFromUrl(String schemaUrl) throws IOException {
+ Configuration conf = new Configuration();
+ Schema.Parser parser = new Schema.Parser();
+ if (schemaUrl.toLowerCase(Locale.ENGLISH).startsWith("hdfs://")) {
+ FileSystem fs = FileSystem.get(conf);
+ FSDataInputStream input = null;
+ try {
+ input = fs.open(new Path(schemaUrl));
+ return parser.parse(input);
+ } finally {
+ if (input != null) {
+ input.close();
+ }
+ }
+ } else {
+ InputStream is = null;
+ try {
+ is = new URL(schemaUrl).openStream();
+ return parser.parse(is);
+ } finally {
+ if (is != null) {
+ is.close();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ dataFileWriter.flush();
+ }
+
+ @Override
+ public void beforeClose() throws IOException {
+ // no-op
+ }
+
+ @Override
+ public boolean supportsReopen() {
+ return false;
+ }
+
+ public static class Builder implements EventSerializer.Builder {
+
+ @Override
+ public EventSerializer build(Context context, OutputStream out) {
+ AvroEventSerializer writer = new AvroEventSerializer(out);
+ writer.configure(context);
+ return writer;
+ }
+
+ }
+
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketClosedException.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketClosedException.java
new file mode 100644
index 0000000..1d8a9e4
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketClosedException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hdfs;
+
+import org.apache.flume.FlumeException;
+
+public class BucketClosedException extends FlumeException {
+
+ private static final long serialVersionUID = -4216667125119540357L;
+
+ public BucketClosedException(String msg) {
+ super(msg);
+ }
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketWriter.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketWriter.java
new file mode 100644
index 0000000..c3941a6
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketWriter.java
@@ -0,0 +1,776 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flume.sink.hdfs;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Throwables;
+import org.apache.flume.Clock;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.SystemClock;
+import org.apache.flume.auth.PrivilegedExecutor;
+import org.apache.flume.instrumentation.SinkCounter;
+import org.apache.flume.sink.hdfs.HDFSEventSink.WriterCallback;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Internal API intended for HDFSSink use.
+ * This class does file rolling and handles file formats and serialization.
+ * Only the public methods in this class are thread safe.
+ */
+class BucketWriter {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(BucketWriter.class);
+
+ /**
+ * This lock ensures that only one thread can open a file at a time.
+ */
+ private static final Integer staticLock = new Integer(1);
+ private Method isClosedMethod = null;
+
+ private final HDFSWriter writer;
+ private final long rollInterval;
+ private final long rollSize;
+ private final long rollCount;
+ private final long batchSize;
+ private final CompressionCodec codeC;
+ private final CompressionType compType;
+ private final ScheduledExecutorService timedRollerPool;
+ private final PrivilegedExecutor proxyUser;
+
+ private final AtomicLong fileExtensionCounter;
+
+ private long eventCounter;
+ private long processSize;
+
+ private FileSystem fileSystem;
+
+ private volatile String filePath;
+ private volatile String fileName;
+ private volatile String inUsePrefix;
+ private volatile String inUseSuffix;
+ private volatile String fileSuffix;
+ private volatile String bucketPath;
+ private volatile String targetPath;
+ private volatile long batchCounter;
+ private volatile boolean isOpen;
+ private volatile boolean isUnderReplicated;
+ private volatile int consecutiveUnderReplRotateCount = 0;
+ private volatile ScheduledFuture<Void> timedRollFuture;
+ private SinkCounter sinkCounter;
+ private final int idleTimeout;
+ private volatile ScheduledFuture<Void> idleFuture;
+ private final WriterCallback onCloseCallback;
+ private final String onCloseCallbackPath;
+ private final long callTimeout;
+ private final ExecutorService callTimeoutPool;
+ private final int maxConsecUnderReplRotations = 30; // make this config'able?
+
+ private boolean mockFsInjected = false;
+
+ private final long retryInterval;
+ private final int maxRetries;
+
+ // flag that the bucket writer was closed due to idling and thus shouldn't be
+ // reopened. Not ideal, but avoids internals of owners
+ protected AtomicBoolean closed = new AtomicBoolean();
+ AtomicInteger renameTries = new AtomicInteger(0);
+
+ BucketWriter(long rollInterval, long rollSize, long rollCount, long batchSize,
+ Context context, String filePath, String fileName, String inUsePrefix,
+ String inUseSuffix, String fileSuffix, CompressionCodec codeC,
+ CompressionType compType, HDFSWriter writer,
+ ScheduledExecutorService timedRollerPool, PrivilegedExecutor proxyUser,
+ SinkCounter sinkCounter, int idleTimeout, WriterCallback onCloseCallback,
+ String onCloseCallbackPath, long callTimeout,
+ ExecutorService callTimeoutPool, long retryInterval,
+ int maxRetries) {
+ this(rollInterval, rollSize, rollCount, batchSize,
+ context, filePath, fileName, inUsePrefix,
+ inUseSuffix, fileSuffix, codeC,
+ compType, writer,
+ timedRollerPool, proxyUser,
+ sinkCounter, idleTimeout, onCloseCallback,
+ onCloseCallbackPath, callTimeout,
+ callTimeoutPool, retryInterval,
+ maxRetries, new SystemClock());
+ }
+
+ BucketWriter(long rollInterval, long rollSize, long rollCount, long batchSize,
+ Context context, String filePath, String fileName, String inUsePrefix,
+ String inUseSuffix, String fileSuffix, CompressionCodec codeC,
+ CompressionType compType, HDFSWriter writer,
+ ScheduledExecutorService timedRollerPool, PrivilegedExecutor proxyUser,
+ SinkCounter sinkCounter, int idleTimeout, WriterCallback onCloseCallback,
+ String onCloseCallbackPath, long callTimeout,
+ ExecutorService callTimeoutPool, long retryInterval,
+ int maxRetries, Clock clock) {
+ this.rollInterval = rollInterval;
+ this.rollSize = rollSize;
+ this.rollCount = rollCount;
+ this.batchSize = batchSize;
+ this.filePath = filePath;
+ this.fileName = fileName;
+ this.inUsePrefix = inUsePrefix;
+ this.inUseSuffix = inUseSuffix;
+ this.fileSuffix = fileSuffix;
+ this.codeC = codeC;
+ this.compType = compType;
+ this.writer = writer;
+ this.timedRollerPool = timedRollerPool;
+ this.proxyUser = proxyUser;
+ this.sinkCounter = sinkCounter;
+ this.idleTimeout = idleTimeout;
+ this.onCloseCallback = onCloseCallback;
+ this.onCloseCallbackPath = onCloseCallbackPath;
+ this.callTimeout = callTimeout;
+ this.callTimeoutPool = callTimeoutPool;
+ fileExtensionCounter = new AtomicLong(clock.currentTimeMillis());
+
+ this.retryInterval = retryInterval;
+ this.maxRetries = maxRetries;
+ isOpen = false;
+ isUnderReplicated = false;
+ this.writer.configure(context);
+ }
+
+ @VisibleForTesting
+ void setFileSystem(FileSystem fs) {
+ this.fileSystem = fs;
+ mockFsInjected = true;
+ }
+
+ /**
+ * Clear the class counters
+ */
+ private void resetCounters() {
+ eventCounter = 0;
+ processSize = 0;
+ batchCounter = 0;
+ }
+
+ private Method getRefIsClosed() {
+ try {
+ return fileSystem.getClass().getMethod("isFileClosed",
+ Path.class);
+ } catch (Exception e) {
+ LOG.info("isFileClosed() is not available in the version of the " +
+ "distributed filesystem being used. " +
+ "Flume will not attempt to re-close files if the close fails " +
+ "on the first attempt");
+ return null;
+ }
+ }
+
+ private Boolean isFileClosed(FileSystem fs, Path tmpFilePath) throws Exception {
+ return (Boolean)(isClosedMethod.invoke(fs, tmpFilePath));
+ }
+
+ /**
+ * open() is called by append()
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ private void open() throws IOException, InterruptedException {
+ if ((filePath == null) || (writer == null)) {
+ throw new IOException("Invalid file settings");
+ }
+
+ final Configuration config = new Configuration();
+ // disable FileSystem JVM shutdown hook
+ config.setBoolean("fs.automatic.close", false);
+
+ // Hadoop is not thread safe when doing certain RPC operations,
+ // including getFileSystem(), when running under Kerberos.
+ // open() must be called by one thread at a time in the JVM.
+ // NOTE: tried synchronizing on the underlying Kerberos principal previously
+ // which caused deadlocks. See FLUME-1231.
+ synchronized (staticLock) {
+ checkAndThrowInterruptedException();
+
+ try {
+ long counter = fileExtensionCounter.incrementAndGet();
+
+ String fullFileName = fileName + "." + counter;
+
+ if (fileSuffix != null && fileSuffix.length() > 0) {
+ fullFileName += fileSuffix;
+ } else if (codeC != null) {
+ fullFileName += codeC.getDefaultExtension();
+ }
+
+ bucketPath = filePath + "/" + inUsePrefix
+ + fullFileName + inUseSuffix;
+ targetPath = filePath + "/" + fullFileName;
+
+ LOG.info("Creating " + bucketPath);
+ callWithTimeout(new CallRunner<Void>() {
+ @Override
+ public Void call() throws Exception {
+ if (codeC == null) {
+ // Need to get reference to FS using above config before underlying
+ // writer does in order to avoid shutdown hook &
+ // IllegalStateExceptions
+ if (!mockFsInjected) {
+ fileSystem = new Path(bucketPath).getFileSystem(config);
+ }
+ writer.open(bucketPath);
+ } else {
+ // need to get reference to FS before writer does to
+ // avoid shutdown hook
+ if (!mockFsInjected) {
+ fileSystem = new Path(bucketPath).getFileSystem(config);
+ }
+ writer.open(bucketPath, codeC, compType);
+ }
+ return null;
+ }
+ });
+ } catch (Exception ex) {
+ sinkCounter.incrementConnectionFailedCount();
+ if (ex instanceof IOException) {
+ throw (IOException) ex;
+ } else {
+ throw Throwables.propagate(ex);
+ }
+ }
+ }
+ isClosedMethod = getRefIsClosed();
+ sinkCounter.incrementConnectionCreatedCount();
+ resetCounters();
+
+ // if time-based rolling is enabled, schedule the roll
+ if (rollInterval > 0) {
+ Callable<Void> action = new Callable<Void>() {
+ public Void call() throws Exception {
+ LOG.debug("Rolling file ({}): Roll scheduled after {} sec elapsed.",
+ bucketPath, rollInterval);
+ try {
+ // Roll the file and remove reference from sfWriters map.
+ close(true);
+ } catch (Throwable t) {
+ LOG.error("Unexpected error", t);
+ }
+ return null;
+ }
+ };
+ timedRollFuture = timedRollerPool.schedule(action, rollInterval,
+ TimeUnit.SECONDS);
+ }
+
+ isOpen = true;
+ }
+
+ /**
+ * Close the file handle and rename the temp file to the permanent filename.
+ * Safe to call multiple times. Logs HDFSWriter.close() exceptions. This
+ * method will not cause the bucket writer to be dereferenced from the HDFS
+ * sink that owns it. This method should be used only when size or count
+ * based rolling closes this file.
+ */
+ public void close() throws InterruptedException {
+ close(false);
+ }
+
+ private CallRunner<Void> createCloseCallRunner() {
+ return new CallRunner<Void>() {
+ @Override
+ public Void call() throws Exception {
+ writer.close(); // could block
+ return null;
+ }
+ };
+ }
+
+ private class CloseHandler implements Callable<Void> {
+ private final String path = bucketPath;
+ private int closeTries = 0;
+
+ @Override
+ public Void call() throws Exception {
+ close(false);
+ return null;
+ }
+
+ /**
+ * Tries to close the writer. Repeats the close if the maximum number
+ * of retries is not reached or an immediate close is not reuqested.
+ * If all close attempts were unsuccessful we try to recover the lease.
+ * @param immediate An immediate close is required
+ */
+ public void close(boolean immediate) {
+ closeTries++;
+ boolean shouldRetry = closeTries < maxRetries && !immediate;
+ try {
+ callWithTimeout(createCloseCallRunner());
+ sinkCounter.incrementConnectionClosedCount();
+ } catch (InterruptedException | IOException e) {
+ LOG.warn("Closing file: " + path + " failed. Will " +
+ "retry again in " + retryInterval + " seconds.", e);
+ if (timedRollerPool != null && !timedRollerPool.isTerminated()) {
+ if (shouldRetry) {
+ timedRollerPool.schedule(this, retryInterval, TimeUnit.SECONDS);
+ }
+ } else {
+ LOG.warn("Cannot retry close any more timedRollerPool is null or terminated");
+ }
+ if (!shouldRetry) {
+ LOG.warn("Unsuccessfully attempted to close " + path + " " +
+ maxRetries + " times. Initializing lease recovery.");
+ sinkCounter.incrementConnectionFailedCount();
+ recoverLease();
+ }
+ }
+ }
+ }
+
+ private class ScheduledRenameCallable implements Callable<Void> {
+ private final String path = bucketPath;
+ private final String finalPath = targetPath;
+ private FileSystem fs = fileSystem;
+ private int renameTries = 1; // one attempt is already done
+
+ @Override
+ public Void call() throws Exception {
+ if (renameTries >= maxRetries) {
+ LOG.warn("Unsuccessfully attempted to rename " + path + " " +
+ maxRetries + " times. File may still be open.");
+ return null;
+ }
+ renameTries++;
+ try {
+ renameBucket(path, finalPath, fs);
+ } catch (Exception e) {
+ LOG.warn("Renaming file: " + path + " failed. Will " +
+ "retry again in " + retryInterval + " seconds.", e);
+ timedRollerPool.schedule(this, retryInterval, TimeUnit.SECONDS);
+ return null;
+ }
+ return null;
+ }
+ }
+
+ /**
+ * Tries to start the lease recovery process for the current bucketPath
+ * if the fileSystem is DistributedFileSystem.
+ * Catches and logs the IOException.
+ */
+ private synchronized void recoverLease() {
+ if (bucketPath != null && fileSystem instanceof DistributedFileSystem) {
+ try {
+ LOG.debug("Starting lease recovery for {}", bucketPath);
+ ((DistributedFileSystem) fileSystem).recoverLease(new Path(bucketPath));
+ } catch (IOException ex) {
+ LOG.warn("Lease recovery failed for {}", bucketPath, ex);
+ }
+ }
+ }
+
+ public void close(boolean callCloseCallback) throws InterruptedException {
+ close(callCloseCallback, false);
+ }
+
+ /**
+ * Close the file handle and rename the temp file to the permanent filename.
+ * Safe to call multiple times. Logs HDFSWriter.close() exceptions.
+ */
+ public void close(boolean callCloseCallback, boolean immediate)
+ throws InterruptedException {
+ if (callCloseCallback) {
+ if (closed.compareAndSet(false, true)) {
+ runCloseAction(); //remove from the cache as soon as possible
+ } else {
+ LOG.warn("This bucketWriter is already closing or closed.");
+ }
+ }
+ doClose(immediate);
+ }
+
+ private synchronized void doClose(boolean immediate)
+ throws InterruptedException {
+ checkAndThrowInterruptedException();
+ try {
+ flush();
+ } catch (IOException e) {
+ LOG.warn("pre-close flush failed", e);
+ }
+
+ LOG.info("Closing {}", bucketPath);
+ if (isOpen) {
+ new CloseHandler().close(immediate);
+ isOpen = false;
+ } else {
+ LOG.info("HDFSWriter is already closed: {}", bucketPath);
+ }
+
+ // NOTE: timed rolls go through this codepath as well as other roll types
+ if (timedRollFuture != null && !timedRollFuture.isDone()) {
+ timedRollFuture.cancel(false); // do not cancel myself if running!
+ timedRollFuture = null;
+ }
+
+ if (idleFuture != null && !idleFuture.isDone()) {
+ idleFuture.cancel(false); // do not cancel myself if running!
+ idleFuture = null;
+ }
+
+ if (bucketPath != null && fileSystem != null) {
+ // could block or throw IOException
+ try {
+ renameBucket(bucketPath, targetPath, fileSystem);
+ } catch (Exception e) {
+ LOG.warn("failed to rename() file (" + bucketPath +
+ "). Exception follows.", e);
+ sinkCounter.incrementConnectionFailedCount();
+ final Callable<Void> scheduledRename = new ScheduledRenameCallable();
+ timedRollerPool.schedule(scheduledRename, retryInterval, TimeUnit.SECONDS);
+ }
+ }
+ }
+
+ /**
+ * flush the data
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public synchronized void flush() throws IOException, InterruptedException {
+ checkAndThrowInterruptedException();
+ if (!isBatchComplete()) {
+ doFlush();
+
+ if (idleTimeout > 0) {
+ // if the future exists and couldn't be cancelled, that would mean it has already run
+ // or been cancelled
+ if (idleFuture == null || idleFuture.cancel(false)) {
+ Callable<Void> idleAction = new Callable<Void>() {
+ public Void call() throws Exception {
+ LOG.info("Closing idle bucketWriter {} at {}", bucketPath,
+ System.currentTimeMillis());
+ if (isOpen) {
+ close(true);
+ }
+ return null;
+ }
+ };
+ idleFuture = timedRollerPool.schedule(idleAction, idleTimeout,
+ TimeUnit.SECONDS);
+ }
+ }
+ }
+ }
+
+ private void runCloseAction() {
+ try {
+ if (onCloseCallback != null) {
+ onCloseCallback.run(onCloseCallbackPath);
+ }
+ } catch (Throwable t) {
+ LOG.error("Unexpected error", t);
+ }
+ }
+
+ /**
+ * doFlush() must only be called by flush()
+ * @throws IOException
+ */
+ private void doFlush() throws IOException, InterruptedException {
+ callWithTimeout(new CallRunner<Void>() {
+ @Override
+ public Void call() throws Exception {
+ writer.sync(); // could block
+ return null;
+ }
+ });
+ batchCounter = 0;
+ }
+
+ /**
+ * Open file handles, write data, update stats, handle file rolling and
+ * batching / flushing. <br />
+ * If the write fails, the file is implicitly closed and then the IOException
+ * is rethrown. <br />
+ * We rotate before append, and not after, so that the active file rolling
+ * mechanism will never roll an empty file. This also ensures that the file
+ * creation time reflects when the first event was written.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public synchronized void append(final Event event)
+ throws IOException, InterruptedException {
+ checkAndThrowInterruptedException();
+ // If idleFuture is not null, cancel it before we move forward to avoid a
+ // close call in the middle of the append.
+ if (idleFuture != null) {
+ idleFuture.cancel(false);
+ // There is still a small race condition - if the idleFuture is already
+ // running, interrupting it can cause HDFS close operation to throw -
+ // so we cannot interrupt it while running. If the future could not be
+ // cancelled, it is already running - wait for it to finish before
+ // attempting to write.
+ if (!idleFuture.isDone()) {
+ try {
+ idleFuture.get(callTimeout, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException ex) {
+ LOG.warn("Timeout while trying to cancel closing of idle file. Idle" +
+ " file close may have failed", ex);
+ } catch (Exception ex) {
+ LOG.warn("Error while trying to cancel closing of idle file. ", ex);
+ }
+ }
+ idleFuture = null;
+ }
+
+ // If the bucket writer was closed due to roll timeout or idle timeout,
+ // force a new bucket writer to be created. Roll count and roll size will
+ // just reuse this one
+ if (!isOpen) {
+ if (closed.get()) {
+ throw new BucketClosedException("This bucket writer was closed and " +
+ "this handle is thus no longer valid");
+ }
+ open();
+ }
+
+ // check if it's time to rotate the file
+ if (shouldRotate()) {
+ boolean doRotate = true;
+
+ if (isUnderReplicated) {
+ if (maxConsecUnderReplRotations > 0 &&
+ consecutiveUnderReplRotateCount >= maxConsecUnderReplRotations) {
+ doRotate = false;
+ if (consecutiveUnderReplRotateCount == maxConsecUnderReplRotations) {
+ LOG.error("Hit max consecutive under-replication rotations ({}); " +
+ "will not continue rolling files under this path due to " +
+ "under-replication", maxConsecUnderReplRotations);
+ }
+ } else {
+ LOG.warn("Block Under-replication detected. Rotating file.");
+ }
+ consecutiveUnderReplRotateCount++;
+ } else {
+ consecutiveUnderReplRotateCount = 0;
+ }
+
+ if (doRotate) {
+ close();
+ open();
+ }
+ }
+
+ // write the event
+ try {
+ sinkCounter.incrementEventDrainAttemptCount();
+ callWithTimeout(new CallRunner<Void>() {
+ @Override
+ public Void call() throws Exception {
+ writer.append(event); // could block
+ return null;
+ }
+ });
+ } catch (IOException e) {
+ LOG.warn("Caught IOException writing to HDFSWriter ({}). Closing file (" +
+ bucketPath + ") and rethrowing exception.",
+ e.getMessage());
+ close(true);
+ throw e;
+ }
+
+ // update statistics
+ processSize += event.getBody().length;
+ eventCounter++;
+ batchCounter++;
+
+ if (batchCounter == batchSize) {
+ flush();
+ }
+ }
+
+ /**
+ * check if time to rotate the file
+ */
+ private boolean shouldRotate() {
+ boolean doRotate = false;
+
+ if (writer.isUnderReplicated()) {
+ this.isUnderReplicated = true;
+ doRotate = true;
+ } else {
+ this.isUnderReplicated = false;
+ }
+
+ if ((rollCount > 0) && (rollCount <= eventCounter)) {
+ LOG.debug("rolling: rollCount: {}, events: {}", rollCount, eventCounter);
+ doRotate = true;
+ }
+
+ if ((rollSize > 0) && (rollSize <= processSize)) {
+ LOG.debug("rolling: rollSize: {}, bytes: {}", rollSize, processSize);
+ doRotate = true;
+ }
+
+ return doRotate;
+ }
+
+ /**
+ * Rename bucketPath file from .tmp to permanent location.
+ */
+ // When this bucket writer is rolled based on rollCount or
+ // rollSize, the same instance is reused for the new file. But if
+ // the previous file was not closed/renamed,
+ // the bucket writer fields no longer point to it and hence need
+ // to be passed in from the thread attempting to close it. Even
+ // when the bucket writer is closed due to close timeout,
+ // this method can get called from the scheduled thread so the
+ // file gets closed later - so an implicit reference to this
+ // bucket writer would still be alive in the Callable instance.
+ private void renameBucket(String bucketPath, String targetPath, final FileSystem fs)
+ throws IOException, InterruptedException {
+ if (bucketPath.equals(targetPath)) {
+ return;
+ }
+
+ final Path srcPath = new Path(bucketPath);
+ final Path dstPath = new Path(targetPath);
+
+ callWithTimeout(new CallRunner<Void>() {
+ @Override
+ public Void call() throws Exception {
+ if (fs.exists(srcPath)) { // could block
+ LOG.info("Renaming " + srcPath + " to " + dstPath);
+ renameTries.incrementAndGet();
+ fs.rename(srcPath, dstPath); // could block
+ }
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public String toString() {
+ return "[ " + this.getClass().getSimpleName() + " targetPath = " + targetPath +
+ ", bucketPath = " + bucketPath + " ]";
+ }
+
+ private boolean isBatchComplete() {
+ return (batchCounter == 0);
+ }
+
+ /**
+ * This method if the current thread has been interrupted and throws an
+ * exception.
+ * @throws InterruptedException
+ */
+ private static void checkAndThrowInterruptedException()
+ throws InterruptedException {
+ if (Thread.currentThread().interrupted()) {
+ throw new InterruptedException("Timed out before HDFS call was made. "
+ + "Your hdfs.callTimeout might be set too low or HDFS calls are "
+ + "taking too long.");
+ }
+ }
+
+ /**
+ * Execute the callable on a separate thread and wait for the completion
+ * for the specified amount of time in milliseconds. In case of timeout
+ * cancel the callable and throw an IOException
+ */
+ private <T> T callWithTimeout(final CallRunner<T> callRunner)
+ throws IOException, InterruptedException {
+ Future<T> future = callTimeoutPool.submit(new Callable<T>() {
+ @Override
+ public T call() throws Exception {
+ return proxyUser.execute(new PrivilegedExceptionAction<T>() {
+ @Override
+ public T run() throws Exception {
+ return callRunner.call();
+ }
+ });
+ }
+ });
+ try {
+ if (callTimeout > 0) {
+ return future.get(callTimeout, TimeUnit.MILLISECONDS);
+ } else {
+ return future.get();
+ }
+ } catch (TimeoutException eT) {
+ future.cancel(true);
+ sinkCounter.incrementConnectionFailedCount();
+ throw new IOException("Callable timed out after " +
+ callTimeout + " ms" + " on file: " + bucketPath, eT);
+ } catch (ExecutionException e1) {
+ sinkCounter.incrementConnectionFailedCount();
+ Throwable cause = e1.getCause();
+ if (cause instanceof IOException) {
+ throw (IOException) cause;
+ } else if (cause instanceof InterruptedException) {
+ throw (InterruptedException) cause;
+ } else if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ } else if (cause instanceof Error) {
+ throw (Error)cause;
+ } else {
+ throw new RuntimeException(e1);
+ }
+ } catch (CancellationException ce) {
+ throw new InterruptedException(
+ "Blocked callable interrupted by rotation event");
+ } catch (InterruptedException ex) {
+ LOG.warn("Unexpected Exception " + ex.getMessage(), ex);
+ throw ex;
+ }
+ }
+
+ /**
+ * Simple interface whose <tt>call</tt> method is called by
+ * {#callWithTimeout} in a new thread inside a
+ * {@linkplain java.security.PrivilegedExceptionAction#run()} call.
+ * @param <T>
+ */
+ private interface CallRunner<T> {
+ T call() throws Exception;
+ }
+
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSCompressedDataStream.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSCompressedDataStream.java
new file mode 100644
index 0000000..80b7cb4
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSCompressedDataStream.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flume.sink.hdfs;
+
+import java.io.IOException;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.serialization.EventSerializer;
+import org.apache.flume.serialization.EventSerializerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.compress.CodecPool;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
+import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.io.compress.DefaultCodec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HDFSCompressedDataStream extends AbstractHDFSWriter {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(HDFSCompressedDataStream.class);
+
+ private FSDataOutputStream fsOut;
+ private CompressionOutputStream cmpOut;
+ private boolean isFinished = false;
+
+ private String serializerType;
+ private Context serializerContext;
+ private EventSerializer serializer;
+ private boolean useRawLocalFileSystem;
+ private Compressor compressor;
+
+ @Override
+ public void configure(Context context) {
+ super.configure(context);
+
+ serializerType = context.getString("serializer", "TEXT");
+ useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
+ false);
+ serializerContext = new Context(
+ context.getSubProperties(EventSerializer.CTX_PREFIX));
+ logger.info("Serializer = " + serializerType + ", UseRawLocalFileSystem = "
+ + useRawLocalFileSystem);
+ }
+
+ @Override
+ public void open(String filePath) throws IOException {
+ DefaultCodec defCodec = new DefaultCodec();
+ CompressionType cType = CompressionType.BLOCK;
+ open(filePath, defCodec, cType);
+ }
+
+ @Override
+ public void open(String filePath, CompressionCodec codec,
+ CompressionType cType) throws IOException {
+ Configuration conf = new Configuration();
+ Path dstPath = new Path(filePath);
+ FileSystem hdfs = dstPath.getFileSystem(conf);
+ if (useRawLocalFileSystem) {
+ if (hdfs instanceof LocalFileSystem) {
+ hdfs = ((LocalFileSystem)hdfs).getRaw();
+ } else {
+ logger.warn("useRawLocalFileSystem is set to true but file system " +
+ "is not of type LocalFileSystem: " + hdfs.getClass().getName());
+ }
+ }
+ boolean appending = false;
+ if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) {
+ fsOut = hdfs.append(dstPath);
+ appending = true;
+ } else {
+ fsOut = hdfs.create(dstPath);
+ }
+ if (compressor == null) {
+ compressor = CodecPool.getCompressor(codec, conf);
+ }
+ cmpOut = codec.createOutputStream(fsOut, compressor);
+ serializer = EventSerializerFactory.getInstance(serializerType,
+ serializerContext, cmpOut);
+ if (appending && !serializer.supportsReopen()) {
+ cmpOut.close();
+ serializer = null;
+ throw new IOException("serializer (" + serializerType
+ + ") does not support append");
+ }
+
+ registerCurrentStream(fsOut, hdfs, dstPath);
+
+ if (appending) {
+ serializer.afterReopen();
+ } else {
+ serializer.afterCreate();
+ }
+ isFinished = false;
+ }
+
+ @Override
+ public void append(Event e) throws IOException {
+ if (isFinished) {
+ cmpOut.resetState();
+ isFinished = false;
+ }
+ serializer.write(e);
+ }
+
+ @Override
+ public void sync() throws IOException {
+ // We must use finish() and resetState() here -- flush() is apparently not
+ // supported by the compressed output streams (it's a no-op).
+ // Also, since resetState() writes headers, avoid calling it without an
+ // additional write/append operation.
+ // Note: There are bugs in Hadoop & JDK w/ pure-java gzip; see HADOOP-8522.
+ serializer.flush();
+ if (!isFinished) {
+ cmpOut.finish();
+ isFinished = true;
+ }
+ fsOut.flush();
+ hflushOrSync(this.fsOut);
+ }
+
+ @Override
+ public void close() throws IOException {
+ serializer.flush();
+ serializer.beforeClose();
+ if (!isFinished) {
+ cmpOut.finish();
+ isFinished = true;
+ }
+ fsOut.flush();
+ hflushOrSync(fsOut);
+ cmpOut.close();
+ if (compressor != null) {
+ CodecPool.returnCompressor(compressor);
+ compressor = null;
+ }
+ unregisterCurrentStream();
+ }
+
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSDataStream.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSDataStream.java
new file mode 100644
index 0000000..c4ad919
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSDataStream.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flume.sink.hdfs;
+
+import java.io.IOException;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.serialization.EventSerializer;
+import org.apache.flume.serialization.EventSerializerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HDFSDataStream extends AbstractHDFSWriter {
+
+ private static final Logger logger = LoggerFactory.getLogger(HDFSDataStream.class);
+
+ private FSDataOutputStream outStream;
+ private String serializerType;
+ private Context serializerContext;
+ private EventSerializer serializer;
+ private boolean useRawLocalFileSystem;
+
+ @Override
+ public void configure(Context context) {
+ super.configure(context);
+
+ serializerType = context.getString("serializer", "TEXT");
+ useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
+ false);
+ serializerContext =
+ new Context(context.getSubProperties(EventSerializer.CTX_PREFIX));
+ logger.info("Serializer = " + serializerType + ", UseRawLocalFileSystem = "
+ + useRawLocalFileSystem);
+ }
+
+ @VisibleForTesting
+ protected FileSystem getDfs(Configuration conf, Path dstPath) throws IOException {
+ return dstPath.getFileSystem(conf);
+ }
+
+ protected void doOpen(Configuration conf, Path dstPath, FileSystem hdfs) throws IOException {
+ if (useRawLocalFileSystem) {
+ if (hdfs instanceof LocalFileSystem) {
+ hdfs = ((LocalFileSystem)hdfs).getRaw();
+ } else {
+ logger.warn("useRawLocalFileSystem is set to true but file system " +
+ "is not of type LocalFileSystem: " + hdfs.getClass().getName());
+ }
+ }
+
+ boolean appending = false;
+ if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) {
+ outStream = hdfs.append(dstPath);
+ appending = true;
+ } else {
+ outStream = hdfs.create(dstPath);
+ }
+
+ serializer = EventSerializerFactory.getInstance(
+ serializerType, serializerContext, outStream);
+ if (appending && !serializer.supportsReopen()) {
+ outStream.close();
+ serializer = null;
+ throw new IOException("serializer (" + serializerType +
+ ") does not support append");
+ }
+
+ // must call superclass to check for replication issues
+ registerCurrentStream(outStream, hdfs, dstPath);
+
+ if (appending) {
+ serializer.afterReopen();
+ } else {
+ serializer.afterCreate();
+ }
+ }
+
+ @Override
+ public void open(String filePath) throws IOException {
+ Configuration conf = new Configuration();
+ Path dstPath = new Path(filePath);
+ FileSystem hdfs = getDfs(conf, dstPath);
+ doOpen(conf, dstPath, hdfs);
+ }
+
+ @Override
+ public void open(String filePath, CompressionCodec codec,
+ CompressionType cType) throws IOException {
+ open(filePath);
+ }
+
+ @Override
+ public void append(Event e) throws IOException {
+ serializer.write(e);
+ }
+
+ @Override
+ public void sync() throws IOException {
+ serializer.flush();
+ outStream.flush();
+ hflushOrSync(outStream);
+ }
+
+ @Override
+ public void close() throws IOException {
+ serializer.flush();
+ serializer.beforeClose();
+ outStream.flush();
+ hflushOrSync(outStream);
+ outStream.close();
+
+ unregisterCurrentStream();
+ }
+
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSEventSink.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSEventSink.java
new file mode 100644
index 0000000..9e64eb5
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSEventSink.java
@@ -0,0 +1,577 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flume.sink.hdfs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.Map.Entry;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.flume.Channel;
+import org.apache.flume.Clock;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.EventDeliveryException;
+import org.apache.flume.SystemClock;
+import org.apache.flume.Transaction;
+import org.apache.flume.auth.FlumeAuthenticationUtil;
+import org.apache.flume.auth.PrivilegedExecutor;
+import org.apache.flume.conf.BatchSizeSupported;
+import org.apache.flume.conf.Configurable;
+import org.apache.flume.formatter.output.BucketPath;
+import org.apache.flume.instrumentation.SinkCounter;
+import org.apache.flume.sink.AbstractSink;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+public class HDFSEventSink extends AbstractSink implements Configurable, BatchSizeSupported {
+ public interface WriterCallback {
+ public void run(String filePath);
+ }
+
+ private static final Logger LOG = LoggerFactory.getLogger(HDFSEventSink.class);
+
+ private static String DIRECTORY_DELIMITER = System.getProperty("file.separator");
+
+ private static final long defaultRollInterval = 30;
+ private static final long defaultRollSize = 1024;
+ private static final long defaultRollCount = 10;
+ private static final String defaultFileName = "FlumeData";
+ private static final String defaultSuffix = "";
+ private static final String defaultInUsePrefix = "";
+ private static final String defaultInUseSuffix = ".tmp";
+ private static final long defaultBatchSize = 100;
+ private static final String defaultFileType = HDFSWriterFactory.SequenceFileType;
+ private static final int defaultMaxOpenFiles = 5000;
+ // Time between close retries, in seconds
+ private static final long defaultRetryInterval = 180;
+ // Retry forever.
+ private static final int defaultTryCount = Integer.MAX_VALUE;
+
+ public static final String IN_USE_SUFFIX_PARAM_NAME = "hdfs.inUseSuffix";
+
+ /**
+ * Default length of time we wait for blocking BucketWriter calls
+ * before timing out the operation. Intended to prevent server hangs.
+ */
+ private static final long defaultCallTimeout = 30000;
+ /**
+ * Default number of threads available for tasks
+ * such as append/open/close/flush with hdfs.
+ * These tasks are done in a separate thread in
+ * the case that they take too long. In which
+ * case we create a new file and move on.
+ */
+ private static final int defaultThreadPoolSize = 10;
+ private static final int defaultRollTimerPoolSize = 1;
+
+ private final HDFSWriterFactory writerFactory;
+ private WriterLinkedHashMap sfWriters;
+
+ private long rollInterval;
+ private long rollSize;
+ private long rollCount;
+ private long batchSize;
+ private int threadsPoolSize;
+ private int rollTimerPoolSize;
+ private CompressionCodec codeC;
+ private CompressionType compType;
+ private String fileType;
+ private String filePath;
+ private String fileName;
+ private String suffix;
+ private String inUsePrefix;
+ private String inUseSuffix;
+ private TimeZone timeZone;
+ private int maxOpenFiles;
+ private ExecutorService callTimeoutPool;
+ private ScheduledExecutorService timedRollerPool;
+
+ private boolean needRounding = false;
+ private int roundUnit = Calendar.SECOND;
+ private int roundValue = 1;
+ private boolean useLocalTime = false;
+
+ private long callTimeout;
+ private Context context;
+ private SinkCounter sinkCounter;
+
+ private volatile int idleTimeout;
+ private Clock clock;
+ private FileSystem mockFs;
+ private HDFSWriter mockWriter;
+ private final Object sfWritersLock = new Object();
+ private long retryInterval;
+ private int tryCount;
+ private PrivilegedExecutor privExecutor;
+
+ /*
+ * Extended Java LinkedHashMap for open file handle LRU queue.
+ * We want to clear the oldest file handle if there are too many open ones.
+ */
+ private static class WriterLinkedHashMap
+ extends LinkedHashMap<String, BucketWriter> {
+
+ private final int maxOpenFiles;
+
+ public WriterLinkedHashMap(int maxOpenFiles) {
+ super(16, 0.75f, true); // stock initial capacity/load, access ordering
+ this.maxOpenFiles = maxOpenFiles;
+ }
+
+ @Override
+ protected boolean removeEldestEntry(Entry<String, BucketWriter> eldest) {
+ if (size() > maxOpenFiles) {
+ // If we have more that max open files, then close the last one and
+ // return true
+ try {
+ eldest.getValue().close();
+ } catch (InterruptedException e) {
+ LOG.warn(eldest.getKey().toString(), e);
+ Thread.currentThread().interrupt();
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ public HDFSEventSink() {
+ this(new HDFSWriterFactory());
+ }
+
+ public HDFSEventSink(HDFSWriterFactory writerFactory) {
+ this.writerFactory = writerFactory;
+ }
+
+ @VisibleForTesting
+ Map<String, BucketWriter> getSfWriters() {
+ return sfWriters;
+ }
+
+ // read configuration and setup thresholds
+ @Override
+ public void configure(Context context) {
+ this.context = context;
+
+ filePath = Preconditions.checkNotNull(
+ context.getString("hdfs.path"), "hdfs.path is required");
+ fileName = context.getString("hdfs.filePrefix", defaultFileName);
+ this.suffix = context.getString("hdfs.fileSuffix", defaultSuffix);
+ inUsePrefix = context.getString("hdfs.inUsePrefix", defaultInUsePrefix);
+ boolean emptyInUseSuffix = context.getBoolean("hdfs.emptyInUseSuffix", false);
+ if (emptyInUseSuffix) {
+ inUseSuffix = "";
+ String tmpInUseSuffix = context.getString(IN_USE_SUFFIX_PARAM_NAME);
+ if (tmpInUseSuffix != null) {
+ LOG.warn("Ignoring parameter " + IN_USE_SUFFIX_PARAM_NAME + " for hdfs sink: " + getName());
+ }
+ } else {
+ inUseSuffix = context.getString(IN_USE_SUFFIX_PARAM_NAME, defaultInUseSuffix);
+ }
+ String tzName = context.getString("hdfs.timeZone");
+ timeZone = tzName == null ? null : TimeZone.getTimeZone(tzName);
+ rollInterval = context.getLong("hdfs.rollInterval", defaultRollInterval);
+ rollSize = context.getLong("hdfs.rollSize", defaultRollSize);
+ rollCount = context.getLong("hdfs.rollCount", defaultRollCount);
+ batchSize = context.getLong("hdfs.batchSize", defaultBatchSize);
+ idleTimeout = context.getInteger("hdfs.idleTimeout", 0);
+ String codecName = context.getString("hdfs.codeC");
+ fileType = context.getString("hdfs.fileType", defaultFileType);
+ maxOpenFiles = context.getInteger("hdfs.maxOpenFiles", defaultMaxOpenFiles);
+ callTimeout = context.getLong("hdfs.callTimeout", defaultCallTimeout);
+ threadsPoolSize = context.getInteger("hdfs.threadsPoolSize",
+ defaultThreadPoolSize);
+ rollTimerPoolSize = context.getInteger("hdfs.rollTimerPoolSize",
+ defaultRollTimerPoolSize);
+ String kerbConfPrincipal = context.getString("hdfs.kerberosPrincipal");
+ String kerbKeytab = context.getString("hdfs.kerberosKeytab");
+ String proxyUser = context.getString("hdfs.proxyUser");
+ tryCount = context.getInteger("hdfs.closeTries", defaultTryCount);
+ if (tryCount <= 0) {
+ LOG.warn("Retry count value : " + tryCount + " is not " +
+ "valid. The sink will try to close the file until the file " +
+ "is eventually closed.");
+ tryCount = defaultTryCount;
+ }
+ retryInterval = context.getLong("hdfs.retryInterval", defaultRetryInterval);
+ if (retryInterval <= 0) {
+ LOG.warn("Retry Interval value: " + retryInterval + " is not " +
+ "valid. If the first close of a file fails, " +
+ "it may remain open and will not be renamed.");
+ tryCount = 1;
+ }
+
+ Preconditions.checkArgument(batchSize > 0, "batchSize must be greater than 0");
+ if (codecName == null) {
+ codeC = null;
+ compType = CompressionType.NONE;
+ } else {
+ codeC = getCodec(codecName);
+ // TODO : set proper compression type
+ compType = CompressionType.BLOCK;
+ }
+
+ // Do not allow user to set fileType DataStream with codeC together
+ // To prevent output file with compress extension (like .snappy)
+ if (fileType.equalsIgnoreCase(HDFSWriterFactory.DataStreamType) && codecName != null) {
+ throw new IllegalArgumentException("fileType: " + fileType +
+ " which does NOT support compressed output. Please don't set codeC" +
+ " or change the fileType if compressed output is desired.");
+ }
+
+ if (fileType.equalsIgnoreCase(HDFSWriterFactory.CompStreamType)) {
+ Preconditions.checkNotNull(codeC, "It's essential to set compress codec"
+ + " when fileType is: " + fileType);
+ }
+
+ // get the appropriate executor
+ this.privExecutor = FlumeAuthenticationUtil.getAuthenticator(
+ kerbConfPrincipal, kerbKeytab).proxyAs(proxyUser);
+
+ needRounding = context.getBoolean("hdfs.round", false);
+
+ if (needRounding) {
+ String unit = context.getString("hdfs.roundUnit", "second");
+ if (unit.equalsIgnoreCase("hour")) {
+ this.roundUnit = Calendar.HOUR_OF_DAY;
+ } else if (unit.equalsIgnoreCase("minute")) {
+ this.roundUnit = Calendar.MINUTE;
+ } else if (unit.equalsIgnoreCase("second")) {
+ this.roundUnit = Calendar.SECOND;
+ } else {
+ LOG.warn("Rounding unit is not valid, please set one of" +
+ "minute, hour, or second. Rounding will be disabled");
+ needRounding = false;
+ }
+ this.roundValue = context.getInteger("hdfs.roundValue", 1);
+ if (roundUnit == Calendar.SECOND || roundUnit == Calendar.MINUTE) {
+ Preconditions.checkArgument(roundValue > 0 && roundValue <= 60,
+ "Round value" +
+ "must be > 0 and <= 60");
+ } else if (roundUnit == Calendar.HOUR_OF_DAY) {
+ Preconditions.checkArgument(roundValue > 0 && roundValue <= 24,
+ "Round value" +
+ "must be > 0 and <= 24");
+ }
+ }
+
+ this.useLocalTime = context.getBoolean("hdfs.useLocalTimeStamp", false);
+ if (useLocalTime) {
+ clock = new SystemClock();
+ }
+
+ if (sinkCounter == null) {
+ sinkCounter = new SinkCounter(getName());
+ }
+ }
+
+ private static boolean codecMatches(Class<? extends CompressionCodec> cls, String codecName) {
+ String simpleName = cls.getSimpleName();
+ if (cls.getName().equals(codecName) || simpleName.equalsIgnoreCase(codecName)) {
+ return true;
+ }
+ if (simpleName.endsWith("Codec")) {
+ String prefix = simpleName.substring(0, simpleName.length() - "Codec".length());
+ if (prefix.equalsIgnoreCase(codecName)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @VisibleForTesting
+ static CompressionCodec getCodec(String codecName) {
+ Configuration conf = new Configuration();
+ List<Class<? extends CompressionCodec>> codecs = CompressionCodecFactory.getCodecClasses(conf);
+ // Wish we could base this on DefaultCodec but appears not all codec's
+ // extend DefaultCodec(Lzo)
+ CompressionCodec codec = null;
+ ArrayList<String> codecStrs = new ArrayList<String>();
+ codecStrs.add("None");
+ for (Class<? extends CompressionCodec> cls : codecs) {
+ codecStrs.add(cls.getSimpleName());
+ if (codecMatches(cls, codecName)) {
+ try {
+ codec = cls.newInstance();
+ } catch (InstantiationException e) {
+ LOG.error("Unable to instantiate " + cls + " class");
+ } catch (IllegalAccessException e) {
+ LOG.error("Unable to access " + cls + " class");
+ }
+ }
+ }
+
+ if (codec == null) {
+ if (!codecName.equalsIgnoreCase("None")) {
+ throw new IllegalArgumentException("Unsupported compression codec "
+ + codecName + ". Please choose from: " + codecStrs);
+ }
+ } else if (codec instanceof org.apache.hadoop.conf.Configurable) {
+ // Must check instanceof codec as BZip2Codec doesn't inherit Configurable
+ // Must set the configuration for Configurable objects that may or do use
+ // native libs
+ ((org.apache.hadoop.conf.Configurable) codec).setConf(conf);
+ }
+ return codec;
+ }
+
+
+ /**
+ * Pull events out of channel and send it to HDFS. Take at most batchSize
+ * events per Transaction. Find the corresponding bucket for the event.
+ * Ensure the file is open. Serialize the data and write it to the file on
+ * HDFS. <br/>
+ * This method is not thread safe.
+ */
+ public Status process() throws EventDeliveryException {
+ Channel channel = getChannel();
+ Transaction transaction = channel.getTransaction();
+ transaction.begin();
+ try {
+ Set<BucketWriter> writers = new LinkedHashSet<>();
+ int txnEventCount = 0;
+ for (txnEventCount = 0; txnEventCount < batchSize; txnEventCount++) {
+ Event event = channel.take();
+ if (event == null) {
+ break;
+ }
+
+ // reconstruct the path name by substituting place holders
+ String realPath = BucketPath.escapeString(filePath, event.getHeaders(),
+ timeZone, needRounding, roundUnit, roundValue, useLocalTime);
+ String realName = BucketPath.escapeString(fileName, event.getHeaders(),
+ timeZone, needRounding, roundUnit, roundValue, useLocalTime);
+
+ String lookupPath = realPath + DIRECTORY_DELIMITER + realName;
+ BucketWriter bucketWriter;
+ HDFSWriter hdfsWriter = null;
+ // Callback to remove the reference to the bucket writer from the
+ // sfWriters map so that all buffers used by the HDFS file
+ // handles are garbage collected.
+ WriterCallback closeCallback = new WriterCallback() {
+ @Override
+ public void run(String bucketPath) {
+ LOG.info("Writer callback called.");
+ synchronized (sfWritersLock) {
+ sfWriters.remove(bucketPath);
+ }
+ }
+ };
+ synchronized (sfWritersLock) {
+ bucketWriter = sfWriters.get(lookupPath);
+ // we haven't seen this file yet, so open it and cache the handle
+ if (bucketWriter == null) {
+ hdfsWriter = writerFactory.getWriter(fileType);
+ bucketWriter = initializeBucketWriter(realPath, realName,
+ lookupPath, hdfsWriter, closeCallback);
+ sfWriters.put(lookupPath, bucketWriter);
+ }
+ }
+
+ // Write the data to HDFS
+ try {
+ bucketWriter.append(event);
+ } catch (BucketClosedException ex) {
+ LOG.info("Bucket was closed while trying to append, " +
+ "reinitializing bucket and writing event.");
+ hdfsWriter = writerFactory.getWriter(fileType);
+ bucketWriter = initializeBucketWriter(realPath, realName,
+ lookupPath, hdfsWriter, closeCallback);
+ synchronized (sfWritersLock) {
+ sfWriters.put(lookupPath, bucketWriter);
+ }
+ bucketWriter.append(event);
+ }
+
+ // track the buckets getting written in this transaction
+ if (!writers.contains(bucketWriter)) {
+ writers.add(bucketWriter);
+ }
+ }
+
+ if (txnEventCount == 0) {
+ sinkCounter.incrementBatchEmptyCount();
+ } else if (txnEventCount == batchSize) {
+ sinkCounter.incrementBatchCompleteCount();
+ } else {
+ sinkCounter.incrementBatchUnderflowCount();
+ }
+
+ // flush all pending buckets before committing the transaction
+ for (BucketWriter bucketWriter : writers) {
+ bucketWriter.flush();
+ }
+
+ transaction.commit();
+
+ if (txnEventCount < 1) {
+ return Status.BACKOFF;
+ } else {
+ sinkCounter.addToEventDrainSuccessCount(txnEventCount);
+ return Status.READY;
+ }
+ } catch (IOException eIO) {
+ transaction.rollback();
+ LOG.warn("HDFS IO error", eIO);
+ sinkCounter.incrementEventWriteFail();
+ return Status.BACKOFF;
+ } catch (Throwable th) {
+ transaction.rollback();
+ LOG.error("process failed", th);
+ sinkCounter.incrementEventWriteOrChannelFail(th);
+ if (th instanceof Error) {
+ throw (Error) th;
+ } else {
+ throw new EventDeliveryException(th);
+ }
+ } finally {
+ transaction.close();
+ }
+ }
+
+ @VisibleForTesting
+ BucketWriter initializeBucketWriter(String realPath,
+ String realName, String lookupPath, HDFSWriter hdfsWriter,
+ WriterCallback closeCallback) {
+ HDFSWriter actualHdfsWriter = mockFs == null ? hdfsWriter : mockWriter;
+ BucketWriter bucketWriter = new BucketWriter(rollInterval,
+ rollSize, rollCount,
+ batchSize, context, realPath, realName, inUsePrefix, inUseSuffix,
+ suffix, codeC, compType, actualHdfsWriter, timedRollerPool,
+ privExecutor, sinkCounter, idleTimeout, closeCallback,
+ lookupPath, callTimeout, callTimeoutPool, retryInterval,
+ tryCount);
+ if (mockFs != null) {
+ bucketWriter.setFileSystem(mockFs);
+ }
+ return bucketWriter;
+ }
+
+ @Override
+ public void stop() {
+ // do not constrain close() calls with a timeout
+ synchronized (sfWritersLock) {
+ for (Entry<String, BucketWriter> entry : sfWriters.entrySet()) {
+ LOG.info("Closing {}", entry.getKey());
+
+ try {
+ entry.getValue().close(false, true);
+ } catch (Exception ex) {
+ LOG.warn("Exception while closing " + entry.getKey() + ". " +
+ "Exception follows.", ex);
+ if (ex instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ }
+
+ // shut down all our thread pools
+ ExecutorService[] toShutdown = { callTimeoutPool, timedRollerPool };
+ for (ExecutorService execService : toShutdown) {
+ execService.shutdown();
+ try {
+ while (execService.isTerminated() == false) {
+ execService.awaitTermination(
+ Math.max(defaultCallTimeout, callTimeout), TimeUnit.MILLISECONDS);
+ }
+ } catch (InterruptedException ex) {
+ LOG.warn("shutdown interrupted on " + execService, ex);
+ }
+ }
+
+ callTimeoutPool = null;
+ timedRollerPool = null;
+
+ synchronized (sfWritersLock) {
+ sfWriters.clear();
+ sfWriters = null;
+ }
+ sinkCounter.stop();
+ super.stop();
+ }
+
+ @Override
+ public void start() {
+ String timeoutName = "hdfs-" + getName() + "-call-runner-%d";
+ callTimeoutPool = Executors.newFixedThreadPool(threadsPoolSize,
+ new ThreadFactoryBuilder().setNameFormat(timeoutName).build());
+
+ String rollerName = "hdfs-" + getName() + "-roll-timer-%d";
+ timedRollerPool = Executors.newScheduledThreadPool(rollTimerPoolSize,
+ new ThreadFactoryBuilder().setNameFormat(rollerName).build());
+
+ this.sfWriters = new WriterLinkedHashMap(maxOpenFiles);
+ sinkCounter.start();
+ super.start();
+ }
+
+ @Override
+ public String toString() {
+ return "{ Sink type:" + getClass().getSimpleName() + ", name:" + getName() +
+ " }";
+ }
+
+ @VisibleForTesting
+ void setBucketClock(Clock clock) {
+ BucketPath.setClock(clock);
+ }
+
+ @VisibleForTesting
+ void setMockFs(FileSystem mockFs) {
+ this.mockFs = mockFs;
+ }
+
+ @VisibleForTesting
+ void setMockWriter(HDFSWriter writer) {
+ this.mockWriter = writer;
+ }
+
+ @VisibleForTesting
+ int getTryCount() {
+ return tryCount;
+ }
+
+ @Override
+ public long getBatchSize() {
+ return batchSize;
+ }
+
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSSequenceFile.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSSequenceFile.java
new file mode 100644
index 0000000..c5430ba
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSSequenceFile.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flume.sink.hdfs;
+
+import java.io.IOException;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HDFSSequenceFile extends AbstractHDFSWriter {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(HDFSSequenceFile.class);
+ private SequenceFile.Writer writer;
+ private String writeFormat;
+ private Context serializerContext;
+ private SequenceFileSerializer serializer;
+ private boolean useRawLocalFileSystem;
+ private FSDataOutputStream outStream = null;
+
+ public HDFSSequenceFile() {
+ writer = null;
+ }
+
+ @Override
+ public void configure(Context context) {
+ super.configure(context);
+
+ // use binary writable serialize by default
+ writeFormat = context.getString("hdfs.writeFormat",
+ SequenceFileSerializerType.Writable.name());
+ useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
+ false);
+ serializerContext = new Context(
+ context.getSubProperties(SequenceFileSerializerFactory.CTX_PREFIX));
+ serializer = SequenceFileSerializerFactory
+ .getSerializer(writeFormat, serializerContext);
+ logger.info("writeFormat = " + writeFormat + ", UseRawLocalFileSystem = "
+ + useRawLocalFileSystem);
+ }
+
+ @Override
+ public void open(String filePath) throws IOException {
+ open(filePath, null, CompressionType.NONE);
+ }
+
+ @Override
+ public void open(String filePath, CompressionCodec codeC,
+ CompressionType compType) throws IOException {
+ Configuration conf = new Configuration();
+ Path dstPath = new Path(filePath);
+ FileSystem hdfs = dstPath.getFileSystem(conf);
+ open(dstPath, codeC, compType, conf, hdfs);
+ }
+
+ protected void open(Path dstPath, CompressionCodec codeC,
+ CompressionType compType, Configuration conf, FileSystem hdfs)
+ throws IOException {
+ if (useRawLocalFileSystem) {
+ if (hdfs instanceof LocalFileSystem) {
+ hdfs = ((LocalFileSystem)hdfs).getRaw();
+ } else {
+ logger.warn("useRawLocalFileSystem is set to true but file system " +
+ "is not of type LocalFileSystem: " + hdfs.getClass().getName());
+ }
+ }
+ if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) {
+ outStream = hdfs.append(dstPath);
+ } else {
+ outStream = hdfs.create(dstPath);
+ }
+ writer = SequenceFile.createWriter(conf, outStream,
+ serializer.getKeyClass(), serializer.getValueClass(), compType, codeC);
+
+ registerCurrentStream(outStream, hdfs, dstPath);
+ }
+
+ @Override
+ public void append(Event e) throws IOException {
+ for (SequenceFileSerializer.Record record : serializer.serialize(e)) {
+ writer.append(record.getKey(), record.getValue());
+ }
+ }
+
+ @Override
+ public void sync() throws IOException {
+ writer.sync();
+ hflushOrSync(outStream);
+ }
+
+ @Override
+ public void close() throws IOException {
+ writer.close();
+ outStream.close();
+ unregisterCurrentStream();
+ }
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextSerializer.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextSerializer.java
new file mode 100644
index 0000000..32fd206
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextSerializer.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.flume.sink.hdfs;
+
+import java.util.Collections;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.LongWritable;
+
+public class HDFSTextSerializer implements SequenceFileSerializer {
+
+ private Text makeText(Event e) {
+ Text textObject = new Text();
+ textObject.set(e.getBody(), 0, e.getBody().length);
+ return textObject;
+ }
+
+ @Override
+ public Class<LongWritable> getKeyClass() {
+ return LongWritable.class;
+ }
+
+ @Override
+ public Class<Text> getValueClass() {
+ return Text.class;
+ }
+
+ @Override
+ public Iterable<Record> serialize(Event e) {
+ Object key = getKey(e);
+ Object value = getValue(e);
+ return Collections.singletonList(new Record(key, value));
+ }
+
+ private Object getKey(Event e) {
+ // Write the data to HDFS
+ String timestamp = e.getHeaders().get("timestamp");
+ long eventStamp;
+
+ if (timestamp == null) {
+ eventStamp = System.currentTimeMillis();
+ } else {
+ eventStamp = Long.valueOf(timestamp);
+ }
+ return new LongWritable(eventStamp);
+ }
+
+ private Object getValue(Event e) {
+ return makeText(e);
+ }
+
+ public static class Builder implements SequenceFileSerializer.Builder {
+
+ @Override
+ public SequenceFileSerializer build(Context context) {
+ return new HDFSTextSerializer();
+ }
+
+ }
+
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableSerializer.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableSerializer.java
new file mode 100644
index 0000000..b25a6ea
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableSerializer.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flume.sink.hdfs;
+
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.LongWritable;
+
+import java.util.Collections;
+
+public class HDFSWritableSerializer implements SequenceFileSerializer {
+
+ private BytesWritable makeByteWritable(Event e) {
+ BytesWritable bytesObject = new BytesWritable();
+ bytesObject.set(e.getBody(), 0, e.getBody().length);
+ return bytesObject;
+ }
+
+ @Override
+ public Class<LongWritable> getKeyClass() {
+ return LongWritable.class;
+ }
+
+ @Override
+ public Class<BytesWritable> getValueClass() {
+ return BytesWritable.class;
+ }
+
+ @Override
+ public Iterable<Record> serialize(Event e) {
+ Object key = getKey(e);
+ Object value = getValue(e);
+ return Collections.singletonList(new Record(key, value));
+ }
+
+ private Object getKey(Event e) {
+ String timestamp = e.getHeaders().get("timestamp");
+ long eventStamp;
+
+ if (timestamp == null) {
+ eventStamp = System.currentTimeMillis();
+ } else {
+ eventStamp = Long.valueOf(timestamp);
+ }
+ return new LongWritable(eventStamp);
+ }
+
+ private Object getValue(Event e) {
+ return makeByteWritable(e);
+ }
+
+ public static class Builder implements SequenceFileSerializer.Builder {
+
+ @Override
+ public SequenceFileSerializer build(Context context) {
+ return new HDFSWritableSerializer();
+ }
+
+ }
+
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriter.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriter.java
new file mode 100644
index 0000000..44a984a
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriter.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flume.sink.hdfs;
+
+import java.io.IOException;
+
+import org.apache.flume.Event;
+import org.apache.flume.annotations.InterfaceAudience;
+import org.apache.flume.annotations.InterfaceStability;
+import org.apache.flume.conf.Configurable;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.compress.CompressionCodec;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface HDFSWriter extends Configurable {
+
+ public void open(String filePath) throws IOException;
+
+ public void open(String filePath, CompressionCodec codec,
+ CompressionType cType) throws IOException;
+
+ public void append(Event e) throws IOException;
+
+ public void sync() throws IOException;
+
+ public void close() throws IOException;
+
+ public boolean isUnderReplicated();
+
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriterFactory.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriterFactory.java
new file mode 100644
index 0000000..a90d536
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWriterFactory.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flume.sink.hdfs;
+
+import java.io.IOException;
+
+public class HDFSWriterFactory {
+ static final String SequenceFileType = "SequenceFile";
+ static final String DataStreamType = "DataStream";
+ static final String CompStreamType = "CompressedStream";
+
+ public HDFSWriterFactory() {
+
+ }
+
+ public HDFSWriter getWriter(String fileType) throws IOException {
+ if (fileType.equalsIgnoreCase(SequenceFileType)) {
+ return new HDFSSequenceFile();
+ } else if (fileType.equalsIgnoreCase(DataStreamType)) {
+ return new HDFSDataStream();
+ } else if (fileType.equalsIgnoreCase(CompStreamType)) {
+ return new HDFSCompressedDataStream();
+ } else {
+ throw new IOException("File type " + fileType + " not supported");
+ }
+ }
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/KerberosUser.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/KerberosUser.java
new file mode 100644
index 0000000..43297e2
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/KerberosUser.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.flume.sink.hdfs;
+
+/**
+ * Simple Pair class used to define a unique (principal, keyTab) combination.
+ */
+public class KerberosUser {
+
+ private final String principal;
+ private final String keyTab;
+
+ public KerberosUser(String principal, String keyTab) {
+ this.principal = principal;
+ this.keyTab = keyTab;
+ }
+
+ public String getPrincipal() {
+ return principal;
+ }
+
+ public String getKeyTab() {
+ return keyTab;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ final KerberosUser other = (KerberosUser) obj;
+ if ((this.principal == null) ?
+ (other.principal != null) :
+ !this.principal.equals(other.principal)) {
+ return false;
+ }
+ if ((this.keyTab == null) ? (other.keyTab != null) : !this.keyTab.equals(other.keyTab)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 7;
+ hash = 41 * hash + (this.principal != null ? this.principal.hashCode() : 0);
+ hash = 41 * hash + (this.keyTab != null ? this.keyTab.hashCode() : 0);
+ return hash;
+ }
+
+ @Override
+ public String toString() {
+ return "{ principal: " + principal + ", keytab: " + keyTab + " }";
+ }
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializer.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializer.java
new file mode 100644
index 0000000..ec2b760
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializer.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flume.sink.hdfs;
+
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+
+public interface SequenceFileSerializer {
+
+ Class<?> getKeyClass();
+
+ Class<?> getValueClass();
+
+ /**
+ * Format the given event into zero, one or more SequenceFile records
+ *
+ * @param e
+ * event
+ * @return a list of records corresponding to the given event
+ */
+ Iterable<Record> serialize(Event e);
+
+ /**
+ * Knows how to construct this output formatter.<br/>
+ * <b>Note: Implementations MUST provide a public a no-arg constructor.</b>
+ */
+ public interface Builder {
+ public SequenceFileSerializer build(Context context);
+ }
+
+ /**
+ * A key-value pair making up a record in an HDFS SequenceFile
+ */
+ public static class Record {
+ private final Object key;
+ private final Object value;
+
+ public Record(Object key, Object value) {
+ this.key = key;
+ this.value = value;
+ }
+
+ public Object getKey() {
+ return key;
+ }
+
+ public Object getValue() {
+ return value;
+ }
+ }
+
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerFactory.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerFactory.java
new file mode 100644
index 0000000..5678836
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerFactory.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hdfs;
+
+import com.google.common.base.Preconditions;
+import org.apache.flume.Context;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SequenceFileSerializerFactory {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(SequenceFileSerializerFactory.class);
+
+ /**
+ * {@link Context} prefix
+ */
+ static final String CTX_PREFIX = "writeFormat.";
+
+ @SuppressWarnings("unchecked")
+ static SequenceFileSerializer getSerializer(String formatType,
+ Context context) {
+
+ Preconditions.checkNotNull(formatType,
+ "serialize type must not be null");
+
+ // try to find builder class in enum of known formatters
+ SequenceFileSerializerType type;
+ try {
+ type = SequenceFileSerializerType.valueOf(formatType);
+ } catch (IllegalArgumentException e) {
+ logger.debug("Not in enum, loading builder class: {}", formatType);
+ type = SequenceFileSerializerType.Other;
+ }
+ Class<? extends SequenceFileSerializer.Builder> builderClass =
+ type.getBuilderClass();
+
+ // handle the case where they have specified their own builder in the config
+ if (builderClass == null) {
+ try {
+ Class c = Class.forName(formatType);
+ if (c != null && SequenceFileSerializer.Builder.class.isAssignableFrom(c)) {
+ builderClass = (Class<? extends SequenceFileSerializer.Builder>) c;
+ } else {
+ logger.error("Unable to instantiate Builder from {}", formatType);
+ return null;
+ }
+ } catch (ClassNotFoundException ex) {
+ logger.error("Class not found: " + formatType, ex);
+ return null;
+ } catch (ClassCastException ex) {
+ logger.error("Class does not extend " +
+ SequenceFileSerializer.Builder.class.getCanonicalName() + ": " +
+ formatType, ex);
+ return null;
+ }
+ }
+
+ // build the builder
+ SequenceFileSerializer.Builder builder;
+ try {
+ builder = builderClass.newInstance();
+ } catch (InstantiationException ex) {
+ logger.error("Cannot instantiate builder: " + formatType, ex);
+ return null;
+ } catch (IllegalAccessException ex) {
+ logger.error("Cannot instantiate builder: " + formatType, ex);
+ return null;
+ }
+
+ return builder.build(context);
+ }
+
+}
diff --git a/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerType.java b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerType.java
new file mode 100644
index 0000000..2ad7689
--- /dev/null
+++ b/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerType.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hdfs;
+
+public enum SequenceFileSerializerType {
+ Writable(HDFSWritableSerializer.Builder.class),
+ Text(HDFSTextSerializer.Builder.class),
+ Other(null);
+
+ private final Class<? extends SequenceFileSerializer.Builder> builderClass;
+
+ SequenceFileSerializerType(Class<? extends SequenceFileSerializer.Builder> builderClass) {
+ this.builderClass = builderClass;
+ }
+
+ public Class<? extends SequenceFileSerializer.Builder> getBuilderClass() {
+ return builderClass;
+ }
+
+}
+
diff --git a/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSBadDataStream.java b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSBadDataStream.java
new file mode 100644
index 0000000..d325233
--- /dev/null
+++ b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSBadDataStream.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.flume.sink.hdfs;
+
+import java.io.IOException;
+import org.apache.flume.Event;
+
+public class HDFSBadDataStream extends HDFSDataStream {
+ public class HDFSBadSeqWriter extends HDFSSequenceFile {
+ @Override
+ public void append(Event e) throws IOException {
+
+ if (e.getHeaders().containsKey("fault")) {
+ throw new IOException("Injected fault");
+ } else if (e.getHeaders().containsKey("slow")) {
+ long waitTime = Long.parseLong(e.getHeaders().get("slow"));
+ try {
+ Thread.sleep(waitTime);
+ } catch (InterruptedException eT) {
+ throw new IOException("append interrupted", eT);
+ }
+ }
+ super.append(e);
+ }
+
+ }
+
+}
diff --git a/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestSeqWriter.java b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestSeqWriter.java
new file mode 100644
index 0000000..f1dadf1
--- /dev/null
+++ b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestSeqWriter.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.flume.sink.hdfs;
+
+import org.apache.flume.Event;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.compress.CompressionCodec;
+
+import java.io.IOException;
+
+public class HDFSTestSeqWriter extends HDFSSequenceFile {
+ protected volatile boolean closed;
+ protected volatile boolean opened;
+
+ private int openCount = 0;
+
+ HDFSTestSeqWriter(int openCount) {
+ this.openCount = openCount;
+ }
+
+ @Override
+ public void open(String filePath, CompressionCodec codeC, CompressionType compType)
+ throws IOException {
+ super.open(filePath, codeC, compType);
+ if (closed) {
+ opened = true;
+ }
+ }
+
+ @Override
+ public void append(Event e) throws IOException {
+
+ if (e.getHeaders().containsKey("fault")) {
+ throw new IOException("Injected fault");
+ } else if (e.getHeaders().containsKey("fault-once")) {
+ e.getHeaders().remove("fault-once");
+ throw new IOException("Injected fault");
+ } else if (e.getHeaders().containsKey("fault-until-reopen")) {
+ // opening first time.
+ if (openCount == 1) {
+ throw new IOException("Injected fault-until-reopen");
+ }
+ } else if (e.getHeaders().containsKey("slow")) {
+ long waitTime = Long.parseLong(e.getHeaders().get("slow"));
+ try {
+ Thread.sleep(waitTime);
+ } catch (InterruptedException eT) {
+ throw new IOException("append interrupted", eT);
+ }
+ }
+
+ super.append(e);
+ }
+
+ @Override
+ public void close() throws IOException {
+ closed = true;
+ super.close();
+ }
+}
diff --git a/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestWriterFactory.java b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestWriterFactory.java
new file mode 100644
index 0000000..70bd9e6
--- /dev/null
+++ b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/HDFSTestWriterFactory.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.flume.sink.hdfs;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class HDFSTestWriterFactory extends HDFSWriterFactory {
+ static final String TestSequenceFileType = "SequenceFile";
+ static final String BadDataStreamType = "DataStream";
+
+ // so we can get a handle to this one in our test.
+ AtomicInteger openCount = new AtomicInteger(0);
+
+ @Override
+ public HDFSWriter getWriter(String fileType) throws IOException {
+ if (fileType == TestSequenceFileType) {
+ return new HDFSTestSeqWriter(openCount.incrementAndGet());
+ } else if (fileType == BadDataStreamType) {
+ return new HDFSBadDataStream();
+ } else {
+ throw new IOException("File type " + fileType + " not supported");
+ }
+ }
+}
diff --git a/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockDataStream.java b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockDataStream.java
new file mode 100644
index 0000000..a85a99f
--- /dev/null
+++ b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockDataStream.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+
+class MockDataStream extends HDFSDataStream {
+ private final FileSystem fs;
+
+ MockDataStream(FileSystem fs) {
+ this.fs = fs;
+ }
+
+ @Override
+ protected FileSystem getDfs(Configuration conf, Path dstPath) throws IOException {
+ return fs;
+ }
+
+}
diff --git a/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFileSystem.java b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFileSystem.java
new file mode 100644
index 0000000..a079b83
--- /dev/null
+++ b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFileSystem.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hdfs;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Progressable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MockFileSystem extends FileSystem {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(MockFileSystem.class);
+
+ FileSystem fs;
+ int numberOfRetriesRequired;
+ MockFsDataOutputStream latestOutputStream;
+ int currentRenameAttempts;
+ boolean closeSucceed = true;
+
+ public MockFileSystem(FileSystem fs, int numberOfRetriesRequired) {
+ this.fs = fs;
+ this.numberOfRetriesRequired = numberOfRetriesRequired;
+ }
+
+ public MockFileSystem(FileSystem fs,
+ int numberOfRetriesRequired, boolean closeSucceed) {
+ this.fs = fs;
+ this.numberOfRetriesRequired = numberOfRetriesRequired;
+ this.closeSucceed = closeSucceed;
+ }
+
+ @Override
+ public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2)
+ throws IOException {
+
+ latestOutputStream = new MockFsDataOutputStream(
+ fs.append(arg0, arg1, arg2), closeSucceed);
+
+ return latestOutputStream;
+ }
+
+ @Override
+ public FSDataOutputStream create(Path arg0) throws IOException {
+ latestOutputStream = new MockFsDataOutputStream(fs.create(arg0), closeSucceed);
+ return latestOutputStream;
+ }
+
+ @Override
+ public FSDataOutputStream create(Path arg0, FsPermission arg1, boolean arg2, int arg3,
+ short arg4, long arg5, Progressable arg6)
+ throws IOException {
+ throw new IOException("Not a real file system");
+ }
+
+ @Override
+ @Deprecated
+ public boolean delete(Path arg0) throws IOException {
+ return fs.delete(arg0);
+ }
+
+ @Override
+ public boolean delete(Path arg0, boolean arg1) throws IOException {
+ return fs.delete(arg0, arg1);
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path arg0) throws IOException {
+ return fs.getFileStatus(arg0);
+ }
+
+ @Override
+ public URI getUri() {
+ return fs.getUri();
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return fs.getWorkingDirectory();
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path arg0) throws IOException {
+ return fs.listStatus(arg0);
+ }
+
+ @Override
+ public boolean mkdirs(Path arg0, FsPermission arg1) throws IOException {
+ // TODO Auto-generated method stub
+ return fs.mkdirs(arg0, arg1);
+ }
+
+ @Override
+ public FSDataInputStream open(Path arg0, int arg1) throws IOException {
+ return fs.open(arg0, arg1);
+ }
+
+ @Override
+ public boolean rename(Path arg0, Path arg1) throws IOException {
+ currentRenameAttempts++;
+ logger.info("Attempting to Rename: '" + currentRenameAttempts + "' of '" +
+ numberOfRetriesRequired + "'");
+ if (currentRenameAttempts >= numberOfRetriesRequired || numberOfRetriesRequired == 0) {
+ logger.info("Renaming file");
+ return fs.rename(arg0, arg1);
+ } else {
+ throw new IOException("MockIOException");
+ }
+ }
+
+ @Override
+ public void setWorkingDirectory(Path arg0) {
+ fs.setWorkingDirectory(arg0);
+ }
+}
diff --git a/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFsDataOutputStream.java b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFsDataOutputStream.java
new file mode 100644
index 0000000..f5d579c
--- /dev/null
+++ b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockFsDataOutputStream.java
@@ -0,0 +1,49 @@
+/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
+package org.apache.flume.sink.hdfs;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class MockFsDataOutputStream extends FSDataOutputStream {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(MockFsDataOutputStream.class);
+
+ boolean closeSucceed;
+
+ public MockFsDataOutputStream(FSDataOutputStream wrapMe, boolean closeSucceed)
+ throws IOException {
+ super(wrapMe.getWrappedStream(), null);
+ this.closeSucceed = closeSucceed;
+ }
+
+ @Override
+ public void close() throws IOException {
+ logger.info("Close Succeeded - " + closeSucceed);
+ if (closeSucceed) {
+ logger.info("closing file");
+ super.close();
+ } else {
+ throw new IOException("MockIOException");
+ }
+ }
+}
diff --git a/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockHDFSWriter.java b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockHDFSWriter.java
new file mode 100644
index 0000000..c57496b
--- /dev/null
+++ b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MockHDFSWriter.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hdfs;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MockHDFSWriter implements HDFSWriter {
+
+ private int filesOpened = 0;
+ private int filesClosed = 0;
+ private int bytesWritten = 0;
+ private int eventsWritten = 0;
+ private String filePath = null;
+ private static final Logger logger = LoggerFactory.getLogger(MockHDFSWriter.class);
+ private int numberOfRetriesRequired;
+ public volatile AtomicInteger currentCloseAttempts = new AtomicInteger(0);
+
+ public MockHDFSWriter(int numberOfRetriesRequired) {
+ this.numberOfRetriesRequired = numberOfRetriesRequired;
+ }
+
+ public MockHDFSWriter() {
+ this.numberOfRetriesRequired = 0;
+ }
+
+ public int getFilesOpened() {
+ return filesOpened;
+ }
+
+ public int getFilesClosed() {
+ return filesClosed;
+ }
+
+ public int getBytesWritten() {
+ return bytesWritten;
+ }
+
+ public int getEventsWritten() {
+ return eventsWritten;
+ }
+
+ public String getOpenedFilePath() {
+ return filePath;
+ }
+
+ public void configure(Context context) {
+ // no-op
+ }
+
+ public void open(String filePath) throws IOException {
+ this.filePath = filePath;
+ filesOpened++;
+ }
+
+ public void open(String filePath, CompressionCodec codec, CompressionType cType)
+ throws IOException {
+ this.filePath = filePath;
+ filesOpened++;
+ }
+
+ public void append(Event e) throws IOException {
+ eventsWritten++;
+ bytesWritten += e.getBody().length;
+ }
+
+ public void sync() throws IOException {
+ // does nothing
+ }
+
+ public void close() throws IOException {
+ filesClosed++;
+ int curr = currentCloseAttempts.incrementAndGet();
+ logger.info("Attempting to close: '" + currentCloseAttempts + "' of '" +
+ numberOfRetriesRequired + "'");
+ if (curr >= numberOfRetriesRequired || numberOfRetriesRequired == 0) {
+ logger.info("closing file");
+ } else {
+ throw new IOException("MockIOException");
+ }
+ }
+
+ @Override
+ public boolean isUnderReplicated() {
+ return false;
+ }
+
+}
diff --git a/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomSerializer.java b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomSerializer.java
new file mode 100644
index 0000000..72164fd
--- /dev/null
+++ b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomSerializer.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.flume.sink.hdfs;
+
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.LongWritable;
+
+import java.util.Arrays;
+
+public class MyCustomSerializer implements SequenceFileSerializer {
+
+ @Override
+ public Class<LongWritable> getKeyClass() {
+ return LongWritable.class;
+ }
+
+ @Override
+ public Class<BytesWritable> getValueClass() {
+ return BytesWritable.class;
+ }
+
+ @Override
+ public Iterable<Record> serialize(Event e) {
+ return Arrays.asList(
+ new Record(new LongWritable(1234L), new BytesWritable(new byte[10])),
+ new Record(new LongWritable(4567L), new BytesWritable(new byte[20]))
+ );
+ }
+
+ public static class Builder implements SequenceFileSerializer.Builder {
+
+ @Override
+ public SequenceFileSerializer build(Context context) {
+ return new MyCustomSerializer();
+ }
+
+ }
+
+}
diff --git a/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestAvroEventSerializer.java b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestAvroEventSerializer.java
new file mode 100644
index 0000000..6b38da2
--- /dev/null
+++ b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestAvroEventSerializer.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hdfs;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileReader;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.generic.GenericRecordBuilder;
+import org.apache.avro.io.BinaryEncoder;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.io.EncoderFactory;
+import org.apache.avro.reflect.ReflectDatumWriter;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.event.EventBuilder;
+import org.apache.flume.serialization.AvroEventSerializerConfigurationConstants;
+import org.apache.flume.serialization.EventSerializer;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.After;
+
+public class TestAvroEventSerializer {
+
+ private File file;
+
+ @Before
+ public void setUp() throws Exception {
+ file = File.createTempFile(getClass().getSimpleName(), "");
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ file.delete();
+ }
+
+ @Test
+ public void testNoCompression() throws IOException {
+ createAvroFile(file, null, false, false);
+ validateAvroFile(file);
+ }
+
+ @Test
+ public void testNullCompression() throws IOException {
+ createAvroFile(file, "null", false, false);
+ validateAvroFile(file);
+ }
+
+ @Test
+ public void testDeflateCompression() throws IOException {
+ createAvroFile(file, "deflate", false, false);
+ validateAvroFile(file);
+ }
+
+ @Test
+ public void testSnappyCompression() throws IOException {
+ createAvroFile(file, "snappy", false, false);
+ validateAvroFile(file);
+ }
+
+ @Test
+ public void testSchemaUrl() throws IOException {
+ createAvroFile(file, null, true, false);
+ validateAvroFile(file);
+ }
+
+ @Test
+ public void testStaticSchemaUrl() throws IOException {
+ createAvroFile(file,null,false, true);
+ validateAvroFile(file);
+ }
+
+ @Test
+ public void testBothUrls() throws IOException {
+ createAvroFile(file,null,true,true);
+ validateAvroFile(file);
+ }
+
+ public void createAvroFile(File file, String codec, boolean useSchemaUrl,
+ boolean useStaticSchemaUrl) throws IOException {
+ // serialize a few events using the reflection-based avro serializer
+ OutputStream out = new FileOutputStream(file);
+
+ Context ctx = new Context();
+ if (codec != null) {
+ ctx.put("compressionCodec", codec);
+ }
+
+ Schema schema = Schema.createRecord("myrecord", null, null, false);
+ schema.setFields(Arrays.asList(new Schema.Field[]{
+ new Schema.Field("message", Schema.create(Schema.Type.STRING), null, null)
+ }));
+ GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
+ File schemaFile = null;
+ if (useSchemaUrl || useStaticSchemaUrl) {
+ schemaFile = File.createTempFile(getClass().getSimpleName(), ".avsc");
+ Files.write(schema.toString(), schemaFile, Charsets.UTF_8);
+ }
+
+ if (useStaticSchemaUrl) {
+ ctx.put(AvroEventSerializerConfigurationConstants.STATIC_SCHEMA_URL,
+ schemaFile.toURI().toURL().toExternalForm());
+ }
+
+ EventSerializer.Builder builder = new AvroEventSerializer.Builder();
+ EventSerializer serializer = builder.build(ctx, out);
+
+ serializer.afterCreate();
+ for (int i = 0; i < 3; i++) {
+ GenericRecord record = recordBuilder.set("message", "Hello " + i).build();
+ Event event = EventBuilder.withBody(serializeAvro(record, schema));
+ if (schemaFile == null && !useSchemaUrl) {
+ event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_LITERAL_HEADER,
+ schema.toString());
+ } else if (useSchemaUrl) {
+ event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_URL_HEADER,
+ schemaFile.toURI().toURL().toExternalForm());
+ }
+ serializer.write(event);
+ }
+ serializer.flush();
+ serializer.beforeClose();
+ out.flush();
+ out.close();
+ if (schemaFile != null ) {
+ schemaFile.delete();
+ }
+
+ }
+
+ private byte[] serializeAvro(Object datum, Schema schema) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ReflectDatumWriter<Object> writer = new ReflectDatumWriter<Object>(schema);
+ BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
+ out.reset();
+ writer.write(datum, encoder);
+ encoder.flush();
+ return out.toByteArray();
+ }
+
+ public void validateAvroFile(File file) throws IOException {
+ // read the events back using GenericRecord
+ DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
+ DataFileReader<GenericRecord> fileReader =
+ new DataFileReader<GenericRecord>(file, reader);
+ GenericRecord record = new GenericData.Record(fileReader.getSchema());
+ int numEvents = 0;
+ while (fileReader.hasNext()) {
+ fileReader.next(record);
+ String bodyStr = record.get("message").toString();
+ System.out.println(bodyStr);
+ numEvents++;
+ }
+ fileReader.close();
+ Assert.assertEquals("Should have found a total of 3 events", 3, numEvents);
+ }
+}
diff --git a/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestBucketWriter.java b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestBucketWriter.java
new file mode 100644
index 0000000..64a89e1
--- /dev/null
+++ b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestBucketWriter.java
@@ -0,0 +1,719 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hdfs;
+
+import com.google.common.base.Charsets;
+import org.apache.flume.Clock;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.SystemClock;
+import org.apache.flume.auth.FlumeAuthenticationUtil;
+import org.apache.flume.auth.PrivilegedExecutor;
+import org.apache.flume.event.EventBuilder;
+import org.apache.flume.instrumentation.SinkCounter;
+import org.apache.flume.sink.hdfs.HDFSEventSink.WriterCallback;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Calendar;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class TestBucketWriter {
+
+ private static Logger logger = LoggerFactory.getLogger(TestBucketWriter.class);
+ private Context ctx = new Context();
+
+ private static ScheduledExecutorService timedRollerPool;
+ private static PrivilegedExecutor proxy;
+
+ @BeforeClass
+ public static void setup() {
+ timedRollerPool = Executors.newSingleThreadScheduledExecutor();
+ proxy = FlumeAuthenticationUtil.getAuthenticator(null, null).proxyAs(null);
+ }
+
+ @AfterClass
+ public static void teardown() throws InterruptedException {
+ timedRollerPool.shutdown();
+ timedRollerPool.awaitTermination(2, TimeUnit.SECONDS);
+ timedRollerPool.shutdownNow();
+ }
+
+ @Test
+ public void testEventCountingRoller() throws IOException, InterruptedException {
+ int maxEvents = 100;
+ MockHDFSWriter hdfsWriter = new MockHDFSWriter();
+ BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
+ .setRollCount(maxEvents).build();
+
+ Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
+ for (int i = 0; i < 1000; i++) {
+ bucketWriter.append(e);
+ }
+
+ logger.info("Number of events written: {}", hdfsWriter.getEventsWritten());
+ logger.info("Number of bytes written: {}", hdfsWriter.getBytesWritten());
+ logger.info("Number of files opened: {}", hdfsWriter.getFilesOpened());
+
+ Assert.assertEquals("events written", 1000, hdfsWriter.getEventsWritten());
+ Assert.assertEquals("bytes written", 3000, hdfsWriter.getBytesWritten());
+ Assert.assertEquals("files opened", 10, hdfsWriter.getFilesOpened());
+ }
+
+ @Test
+ public void testSizeRoller() throws IOException, InterruptedException {
+ int maxBytes = 300;
+ MockHDFSWriter hdfsWriter = new MockHDFSWriter();
+ BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
+ .setRollSize(maxBytes).build();
+
+ Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
+ for (int i = 0; i < 1000; i++) {
+ bucketWriter.append(e);
+ }
+
+ logger.info("Number of events written: {}", hdfsWriter.getEventsWritten());
+ logger.info("Number of bytes written: {}", hdfsWriter.getBytesWritten());
+ logger.info("Number of files opened: {}", hdfsWriter.getFilesOpened());
+
+ Assert.assertEquals("events written", 1000, hdfsWriter.getEventsWritten());
+ Assert.assertEquals("bytes written", 3000, hdfsWriter.getBytesWritten());
+ Assert.assertEquals("files opened", 10, hdfsWriter.getFilesOpened());
+ }
+
+ @Test
+ public void testIntervalRoller() throws IOException, InterruptedException {
+ final int ROLL_INTERVAL = 1; // seconds
+ final int NUM_EVENTS = 10;
+ final AtomicBoolean calledBack = new AtomicBoolean(false);
+
+ MockHDFSWriter hdfsWriter = new MockHDFSWriter();
+ BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
+ .setRollInterval(ROLL_INTERVAL)
+ .setOnCloseCallback(new HDFSEventSink.WriterCallback() {
+ @Override
+ public void run(String filePath) {
+ calledBack.set(true);
+ }
+ }).build();
+
+ Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
+ long startNanos = System.nanoTime();
+ for (int i = 0; i < NUM_EVENTS - 1; i++) {
+ bucketWriter.append(e);
+ }
+
+ // sleep to force a roll... wait 2x interval just to be sure
+ Thread.sleep(2 * ROLL_INTERVAL * 1000L);
+
+ Assert.assertTrue(bucketWriter.closed.get());
+ Assert.assertTrue(calledBack.get());
+
+ bucketWriter = new BucketWriterBuilder(hdfsWriter)
+ .setRollInterval(ROLL_INTERVAL).build();
+ // write one more event (to reopen a new file so we will roll again later)
+ bucketWriter.append(e);
+
+ long elapsedMillis = TimeUnit.MILLISECONDS.convert(
+ System.nanoTime() - startNanos, TimeUnit.NANOSECONDS);
+ long elapsedSeconds = elapsedMillis / 1000L;
+
+ logger.info("Time elapsed: {} milliseconds", elapsedMillis);
+ logger.info("Number of events written: {}", hdfsWriter.getEventsWritten());
+ logger.info("Number of bytes written: {}", hdfsWriter.getBytesWritten());
+ logger.info("Number of files opened: {}", hdfsWriter.getFilesOpened());
+ logger.info("Number of files closed: {}", hdfsWriter.getFilesClosed());
+
+ Assert.assertEquals("events written", NUM_EVENTS,
+ hdfsWriter.getEventsWritten());
+ Assert.assertEquals("bytes written", e.getBody().length * NUM_EVENTS,
+ hdfsWriter.getBytesWritten());
+ Assert.assertEquals("files opened", 2, hdfsWriter.getFilesOpened());
+
+ // before auto-roll
+ Assert.assertEquals("files closed", 1, hdfsWriter.getFilesClosed());
+
+ logger.info("Waiting for roll...");
+ Thread.sleep(2 * ROLL_INTERVAL * 1000L);
+
+ logger.info("Number of files closed: {}", hdfsWriter.getFilesClosed());
+ Assert.assertEquals("files closed", 2, hdfsWriter.getFilesClosed());
+ }
+
+ @Test
+ public void testIntervalRollerBug() throws IOException, InterruptedException {
+ final int ROLL_INTERVAL = 1; // seconds
+ final int NUM_EVENTS = 10;
+
+ HDFSWriter hdfsWriter = new HDFSWriter() {
+ private volatile boolean open = false;
+
+ public void configure(Context context) {
+ }
+
+ public void sync() throws IOException {
+ if (!open) {
+ throw new IOException("closed");
+ }
+ }
+
+ public void open(String filePath, CompressionCodec codec, CompressionType cType)
+ throws IOException {
+ open = true;
+ }
+
+ public void open(String filePath) throws IOException {
+ open = true;
+ }
+
+ public void close() throws IOException {
+ open = false;
+ }
+
+ @Override
+ public boolean isUnderReplicated() {
+ return false;
+ }
+
+ public void append(Event e) throws IOException {
+ // we just re-open in append if closed
+ open = true;
+ }
+ };
+
+
+ File tmpFile = File.createTempFile("flume", "test");
+ tmpFile.deleteOnExit();
+ String path = tmpFile.getParent();
+ String name = tmpFile.getName();
+
+ BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
+ .setRollInterval(ROLL_INTERVAL)
+ .setFilePath(path)
+ .setFileName(name)
+ .build();
+
+ Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
+ for (int i = 0; i < NUM_EVENTS - 1; i++) {
+ bucketWriter.append(e);
+ }
+
+ // sleep to force a roll... wait 2x interval just to be sure
+ Thread.sleep(2 * ROLL_INTERVAL * 1000L);
+
+ bucketWriter.flush(); // throws closed exception
+ }
+
+ @Test
+ public void testFileSuffixNotGiven() throws IOException, InterruptedException {
+ final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
+ final String suffix = null;
+
+ // Need to override system time use for test so we know what to expect
+ final long testTime = System.currentTimeMillis();
+ Clock testClock = new Clock() {
+ public long currentTimeMillis() {
+ return testTime;
+ }
+ };
+
+ MockHDFSWriter hdfsWriter = new MockHDFSWriter();
+ BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
+ .setRollInterval(ROLL_INTERVAL)
+ .setFileSuffix(suffix)
+ .setClock(testClock)
+ .build();
+
+ Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
+ bucketWriter.append(e);
+
+ Assert.assertTrue("Incorrect suffix", hdfsWriter.getOpenedFilePath().endsWith(
+ Long.toString(testTime + 1) + ".tmp"));
+ }
+
+ @Test
+ public void testFileSuffixGiven() throws IOException, InterruptedException {
+ final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
+ final String suffix = ".avro";
+
+ // Need to override system time use for test so we know what to expect
+
+ final long testTime = System.currentTimeMillis();
+
+ Clock testClock = new Clock() {
+ public long currentTimeMillis() {
+ return testTime;
+ }
+ };
+
+ MockHDFSWriter hdfsWriter = new MockHDFSWriter();
+ BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
+ .setRollInterval(ROLL_INTERVAL)
+ .setFileSuffix(suffix)
+ .setClock(testClock)
+ .build();
+
+
+ Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
+ bucketWriter.append(e);
+
+ Assert.assertTrue("Incorrect suffix",hdfsWriter.getOpenedFilePath().endsWith(
+ Long.toString(testTime + 1) + suffix + ".tmp"));
+ }
+
+ @Test
+ public void testFileSuffixCompressed()
+ throws IOException, InterruptedException {
+ final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
+ final String suffix = ".foo";
+
+ MockHDFSWriter hdfsWriter = new MockHDFSWriter();
+
+ // Need to override system time use for test so we know what to expect
+ final long testTime = System.currentTimeMillis();
+
+ Clock testClock = new Clock() {
+ public long currentTimeMillis() {
+ return testTime;
+ }
+ };
+
+ BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
+ .setRollInterval(ROLL_INTERVAL)
+ .setFileSuffix(suffix)
+ .setCodeC(HDFSEventSink.getCodec("gzip"))
+ .setCompType(SequenceFile.CompressionType.BLOCK)
+ .setClock(testClock)
+ .build();
+
+
+
+
+ Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
+ bucketWriter.append(e);
+
+ Assert.assertTrue("Incorrect suffix", hdfsWriter.getOpenedFilePath().endsWith(
+ Long.toString(testTime + 1) + suffix + ".tmp"));
+ }
+
+ @Test
+ public void testInUsePrefix() throws IOException, InterruptedException {
+ final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
+ final String PREFIX = "BRNO_IS_CITY_IN_CZECH_REPUBLIC";
+
+ MockHDFSWriter hdfsWriter = new MockHDFSWriter();
+ HDFSTextSerializer formatter = new HDFSTextSerializer();
+ BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
+ .setRollInterval(ROLL_INTERVAL)
+ .setInUsePrefix(PREFIX)
+ .build();
+
+ Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
+ bucketWriter.append(e);
+
+ Assert.assertTrue("Incorrect in use prefix", hdfsWriter.getOpenedFilePath().contains(PREFIX));
+ }
+
+ @Test
+ public void testInUseSuffix() throws IOException, InterruptedException {
+ final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
+ final String SUFFIX = "WELCOME_TO_THE_HELLMOUNTH";
+
+ MockHDFSWriter hdfsWriter = new MockHDFSWriter();
+ HDFSTextSerializer serializer = new HDFSTextSerializer();
+ BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
+ .setRollInterval(ROLL_INTERVAL)
+ .setInUseSuffix(SUFFIX)
+ .build();
+
+ Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
+ bucketWriter.append(e);
+
+ Assert.assertTrue("Incorrect in use suffix", hdfsWriter.getOpenedFilePath().contains(SUFFIX));
+ }
+
+ @Test
+ public void testCallbackOnClose() throws IOException, InterruptedException {
+ final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
+ final String SUFFIX = "WELCOME_TO_THE_EREBOR";
+ final AtomicBoolean callbackCalled = new AtomicBoolean(false);
+
+ BucketWriter bucketWriter = new BucketWriterBuilder()
+ .setRollInterval(ROLL_INTERVAL)
+ .setInUseSuffix(SUFFIX)
+ .setOnCloseCallback(new HDFSEventSink.WriterCallback() {
+ @Override
+ public void run(String filePath) {
+ callbackCalled.set(true);
+ }
+ })
+ .setOnCloseCallbackPath("blah")
+ .build();
+
+ Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
+ bucketWriter.append(e);
+ bucketWriter.close(true);
+
+ Assert.assertTrue(callbackCalled.get());
+ }
+
+ @Test
+ public void testSequenceFileRenameRetries() throws Exception {
+ sequenceFileRenameRetryCoreTest(1, true);
+ sequenceFileRenameRetryCoreTest(5, true);
+ sequenceFileRenameRetryCoreTest(2, true);
+
+ sequenceFileRenameRetryCoreTest(1, false);
+ sequenceFileRenameRetryCoreTest(5, false);
+ sequenceFileRenameRetryCoreTest(2, false);
+ }
+
+ @Test
+ public void testSequenceFileCloseRetries() throws Exception {
+ sequenceFileCloseRetryCoreTest(5);
+ sequenceFileCloseRetryCoreTest(1);
+ }
+
+ public void sequenceFileRenameRetryCoreTest(int numberOfRetriesRequired, boolean closeSucceed)
+ throws Exception {
+ String hdfsPath = "file:///tmp/flume-test." +
+ Calendar.getInstance().getTimeInMillis() +
+ "." + Thread.currentThread().getId();
+
+ Context context = new Context();
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ Path dirPath = new Path(hdfsPath);
+ fs.delete(dirPath, true);
+ fs.mkdirs(dirPath);
+ context.put("hdfs.path", hdfsPath);
+ context.put("hdfs.closeTries", String.valueOf(numberOfRetriesRequired));
+ context.put("hdfs.rollCount", "1");
+ context.put("hdfs.retryInterval", "1");
+ context.put("hdfs.callTimeout", Long.toString(1000));
+ MockFileSystem mockFs = new MockFileSystem(fs, numberOfRetriesRequired, closeSucceed);
+ MockDataStream writer = new MockDataStream(mockFs);
+ BucketWriter bucketWriter = new BucketWriterBuilder(writer)
+ .setRollCount(1)
+ .setBatchSize(1)
+ .setFilePath(hdfsPath)
+ .setFileName(hdfsPath)
+ .setInUsePrefix("singleBucket")
+ .setCompType(null)
+ .setRetryInterval(1)
+ .setMaxCloseTries(numberOfRetriesRequired)
+ .setWriter(writer)
+ .build();
+
+ bucketWriter.setFileSystem(mockFs);
+ // At this point, we checked if isFileClosed is available in
+ // this JVM, so lets make it check again.
+ Event event = EventBuilder.withBody("test", Charsets.UTF_8);
+ bucketWriter.append(event);
+ // This is what triggers the close, so a 2nd append is required :/
+ bucketWriter.append(event);
+
+ TimeUnit.SECONDS.sleep(numberOfRetriesRequired + 2);
+
+ Assert.assertTrue("Expected " + numberOfRetriesRequired + " " +
+ "but got " + bucketWriter.renameTries.get(),
+ bucketWriter.renameTries.get() == numberOfRetriesRequired);
+ }
+
+ private void sequenceFileCloseRetryCoreTest(int numberOfRetriesRequired)
+ throws Exception {
+ String hdfsPath = "file:///tmp/flume-test." +
+ Calendar.getInstance().getTimeInMillis() +
+ "." + Thread.currentThread().getId();
+
+ Context context = new Context();
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ Path dirPath = new Path(hdfsPath);
+ fs.delete(dirPath, true);
+ fs.mkdirs(dirPath);
+ context.put("hdfs.path", hdfsPath);
+ context.put("hdfs.closeTries", String.valueOf(numberOfRetriesRequired));
+ context.put("hdfs.rollCount", "1");
+ context.put("hdfs.retryInterval", "1");
+ context.put("hdfs.callTimeout", Long.toString(1000));
+ MockHDFSWriter mockHDFSWriter = new MockHDFSWriter(Integer.MAX_VALUE);
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ BucketWriter bucketWriter = new BucketWriter(
+ 0, 0, 1, 1, ctx, hdfsPath, hdfsPath, "singleBucket", ".tmp", null, null,
+ null, mockHDFSWriter, timedRollerPool, proxy,
+ new SinkCounter("test-bucket-writer-" + System.currentTimeMillis()), 0, null, null, 30000,
+ executorService, 1, numberOfRetriesRequired);
+
+ Event event = EventBuilder.withBody("test", Charsets.UTF_8);
+ bucketWriter.append(event);
+ bucketWriter.close(false);
+ TimeUnit.SECONDS.sleep(numberOfRetriesRequired + 2);
+ Assert.assertEquals("ExcceutorService should be empty",
+ executorService.shutdownNow().size(), 0);
+ Assert.assertEquals("Expected " + numberOfRetriesRequired + " " +
+ "but got " + mockHDFSWriter.currentCloseAttempts,
+ mockHDFSWriter.currentCloseAttempts.get(), numberOfRetriesRequired);
+ }
+
+ // Test that we don't swallow IOExceptions in secure mode. We should close the bucket writer
+ // and rethrow the exception. Regression test for FLUME-3049.
+ @Test
+ public void testRotateBucketOnIOException() throws IOException, InterruptedException {
+ MockHDFSWriter hdfsWriter = Mockito.spy(new MockHDFSWriter());
+ PrivilegedExecutor ugiProxy =
+ FlumeAuthenticationUtil.getAuthenticator(null, null).proxyAs("alice");
+
+ final int ROLL_COUNT = 1; // Cause a roll after every successful append().
+ BucketWriter bucketWriter = new BucketWriterBuilder(hdfsWriter)
+ .setProxyUser(ugiProxy)
+ .setRollCount(ROLL_COUNT)
+ .build();
+
+ Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
+
+ // Write one event successfully.
+ bucketWriter.append(e);
+
+ // Fail the next write.
+ IOException expectedIOException = new IOException("Test injected IOException");
+ Mockito.doThrow(expectedIOException).when(hdfsWriter)
+ .append(Mockito.any(Event.class));
+
+ // The second time we try to write we should get an IOException.
+ try {
+ bucketWriter.append(e);
+ Assert.fail("Expected IOException wasn't thrown during append");
+ } catch (IOException ex) {
+ Assert.assertEquals(expectedIOException, ex);
+ logger.info("Caught expected IOException", ex);
+ }
+
+ // The third time we try to write we should get a BucketClosedException, because the
+ // BucketWriter should attempt to close itself before rethrowing the IOException on the first
+ // call.
+ try {
+ bucketWriter.append(e);
+ Assert.fail("BucketWriter should be already closed, BucketClosedException expected");
+ } catch (BucketClosedException ex) {
+ logger.info("Caught expected BucketClosedException", ex);
+ }
+
+ Assert.assertEquals("events written", 1, hdfsWriter.getEventsWritten());
+ Assert.assertEquals("2 files should be closed", 2, hdfsWriter.getFilesClosed());
+ }
+
+ private class BucketWriterBuilder {
+ private long rollInterval = 0;
+ private long rollSize = 0;
+ private long rollCount = 0;
+ private long batchSize = 0;
+ private Context context = TestBucketWriter.this.ctx;
+ private String filePath = "/tmp";
+ private String fileName = "file";
+ private String inUsePrefix = "";
+ private String inUseSuffix = ".tmp";
+ private String fileSuffix = null;
+ private CompressionCodec codeC = null;
+ private CompressionType compType = SequenceFile.CompressionType.NONE;
+ private HDFSWriter writer = null;
+ private ScheduledExecutorService timedRollerPool = TestBucketWriter.timedRollerPool;
+ private PrivilegedExecutor proxyUser = TestBucketWriter.proxy;
+ private SinkCounter sinkCounter = new SinkCounter(
+ "test-bucket-writer-" + System.currentTimeMillis());
+ private int idleTimeout = 0;
+ private WriterCallback onCloseCallback = null;
+ private String onCloseCallbackPath = null;
+ private long callTimeout = 30000;
+ private ExecutorService callTimeoutPool = Executors.newSingleThreadExecutor();
+ private long retryInterval = 0;
+ private int maxCloseTries = 0;
+ private Clock clock = null;
+
+ public BucketWriterBuilder() {
+ }
+
+ public BucketWriterBuilder(HDFSWriter writer) {
+ this.writer = writer;
+ }
+
+ public BucketWriterBuilder setRollInterval(long rollInterval) {
+ this.rollInterval = rollInterval;
+ return this;
+ }
+
+ public BucketWriterBuilder setRollSize(long rollSize) {
+ this.rollSize = rollSize;
+ return this;
+ }
+
+ public BucketWriterBuilder setRollCount(long rollCount) {
+ this.rollCount = rollCount;
+ return this;
+ }
+
+ public BucketWriterBuilder setBatchSize(long batchSize) {
+ this.batchSize = batchSize;
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public BucketWriterBuilder setContext(Context context) {
+ this.context = context;
+ return this;
+ }
+
+ public BucketWriterBuilder setFilePath(String filePath) {
+ this.filePath = filePath;
+ return this;
+ }
+
+ public BucketWriterBuilder setFileName(String fileName) {
+ this.fileName = fileName;
+ return this;
+ }
+
+ public BucketWriterBuilder setInUsePrefix(String inUsePrefix) {
+ this.inUsePrefix = inUsePrefix;
+ return this;
+ }
+
+ public BucketWriterBuilder setInUseSuffix(String inUseSuffix) {
+ this.inUseSuffix = inUseSuffix;
+ return this;
+ }
+
+ public BucketWriterBuilder setFileSuffix(String fileSuffix) {
+ this.fileSuffix = fileSuffix;
+ return this;
+ }
+
+ public BucketWriterBuilder setCodeC(CompressionCodec codeC) {
+ this.codeC = codeC;
+ return this;
+ }
+
+ public BucketWriterBuilder setCompType(CompressionType compType) {
+ this.compType = compType;
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public BucketWriterBuilder setTimedRollerPool(
+ ScheduledExecutorService timedRollerPool) {
+ this.timedRollerPool = timedRollerPool;
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public BucketWriterBuilder setProxyUser(PrivilegedExecutor proxyUser) {
+ this.proxyUser = proxyUser;
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public BucketWriterBuilder setSinkCounter(SinkCounter sinkCounter) {
+ this.sinkCounter = sinkCounter;
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public BucketWriterBuilder setIdleTimeout(int idleTimeout) {
+ this.idleTimeout = idleTimeout;
+ return this;
+ }
+
+ public BucketWriterBuilder setOnCloseCallback(
+ WriterCallback onCloseCallback) {
+ this.onCloseCallback = onCloseCallback;
+ return this;
+ }
+
+ public BucketWriterBuilder setOnCloseCallbackPath(
+ String onCloseCallbackPath) {
+ this.onCloseCallbackPath = onCloseCallbackPath;
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public BucketWriterBuilder setCallTimeout(long callTimeout) {
+ this.callTimeout = callTimeout;
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public BucketWriterBuilder setCallTimeoutPool(
+ ExecutorService callTimeoutPool) {
+ this.callTimeoutPool = callTimeoutPool;
+ return this;
+ }
+
+ public BucketWriterBuilder setRetryInterval(long retryInterval) {
+ this.retryInterval = retryInterval;
+ return this;
+ }
+
+ public BucketWriterBuilder setMaxCloseTries(int maxCloseTries) {
+ this.maxCloseTries = maxCloseTries;
+ return this;
+ }
+
+ public BucketWriterBuilder setWriter(HDFSWriter writer) {
+ this.writer = writer;
+ return this;
+ }
+
+ public BucketWriterBuilder setClock(Clock clock) {
+ this.clock = clock;
+ return this;
+ }
+
+ public BucketWriter build() {
+ if (clock == null) {
+ clock = new SystemClock();
+ }
+ if (writer == null) {
+ writer = new MockHDFSWriter();
+ }
+
+ return new BucketWriter(rollInterval, rollSize, rollCount, batchSize,
+ context, filePath, fileName, inUsePrefix, inUseSuffix, fileSuffix,
+ codeC, compType, writer, timedRollerPool, proxyUser, sinkCounter,
+ idleTimeout, onCloseCallback, onCloseCallbackPath, callTimeout,
+ callTimeoutPool, retryInterval, maxCloseTries, clock);
+ }
+ }
+}
diff --git a/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSCompressedDataStream.java b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSCompressedDataStream.java
new file mode 100644
index 0000000..540d4bb
--- /dev/null
+++ b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSCompressedDataStream.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flume.sink.hdfs;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.CharsetDecoder;
+import java.util.List;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.avro.file.DataFileStream;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumReader;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.event.EventBuilder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.Lists;
+
+public class TestHDFSCompressedDataStream {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(TestHDFSCompressedDataStream.class);
+
+ private File file;
+ private String fileURI;
+ private CompressionCodecFactory factory;
+
+ @Before
+ public void init() throws Exception {
+ this.file = new File("target/test/data/foo.gz");
+ this.fileURI = file.getAbsoluteFile().toURI().toString();
+ logger.info("File URI: {}", fileURI);
+
+ Configuration conf = new Configuration();
+ // local FS must be raw in order to be Syncable
+ conf.set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem");
+ Path path = new Path(fileURI);
+ path.getFileSystem(conf); // get FS with our conf cached
+
+ this.factory = new CompressionCodecFactory(conf);
+ }
+
+ // make sure the data makes it to disk if we sync() the data stream
+ @Test
+ public void testGzipDurability() throws Exception {
+ Context context = new Context();
+ HDFSCompressedDataStream writer = new HDFSCompressedDataStream();
+ writer.configure(context);
+ writer.open(fileURI, factory.getCodec(new Path(fileURI)),
+ SequenceFile.CompressionType.BLOCK);
+
+ String[] bodies = { "yarf!" };
+ writeBodies(writer, bodies);
+ writer.close();
+
+ byte[] buf = new byte[256];
+ GZIPInputStream cmpIn = new GZIPInputStream(new FileInputStream(file));
+ int len = cmpIn.read(buf);
+ String result = new String(buf, 0, len, Charsets.UTF_8);
+ result = result.trim(); // BodyTextEventSerializer adds a newline
+
+ Assert.assertEquals("input and output must match", bodies[0], result);
+ }
+
+ @Test
+ public void testGzipDurabilityWithSerializer() throws Exception {
+ Context context = new Context();
+ context.put("serializer", "AVRO_EVENT");
+
+ HDFSCompressedDataStream writer = new HDFSCompressedDataStream();
+ writer.configure(context);
+
+ writer.open(fileURI, factory.getCodec(new Path(fileURI)),
+ SequenceFile.CompressionType.BLOCK);
+
+ String[] bodies = { "yarf!", "yarfing!" };
+ writeBodies(writer, bodies);
+ writer.close();
+
+ int found = 0;
+ int expected = bodies.length;
+ List<String> expectedBodies = Lists.newArrayList(bodies);
+
+ GZIPInputStream cmpIn = new GZIPInputStream(new FileInputStream(file));
+ DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
+ DataFileStream<GenericRecord> avroStream =
+ new DataFileStream<GenericRecord>(cmpIn, reader);
+ GenericRecord record = new GenericData.Record(avroStream.getSchema());
+ while (avroStream.hasNext()) {
+ avroStream.next(record);
+ CharsetDecoder decoder = Charsets.UTF_8.newDecoder();
+ String bodyStr = decoder.decode((ByteBuffer) record.get("body"))
+ .toString();
+ expectedBodies.remove(bodyStr);
+ found++;
+ }
+ avroStream.close();
+ cmpIn.close();
+
+ Assert.assertTrue("Found = " + found + ", Expected = " + expected
+ + ", Left = " + expectedBodies.size() + " " + expectedBodies,
+ expectedBodies.size() == 0);
+ }
+
+ private void writeBodies(HDFSCompressedDataStream writer, String... bodies)
+ throws Exception {
+ for (String body : bodies) {
+ Event evt = EventBuilder.withBody(body, Charsets.UTF_8);
+ writer.append(evt);
+ }
+ writer.sync();
+ }
+}
diff --git a/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSink.java b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSink.java
new file mode 100644
index 0000000..25bd2da
--- /dev/null
+++ b/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSink.java
@@ -0,0 +1,1687 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flume.sink.hdfs;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+import java.nio.charset.CharsetDecoder;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Maps;
+import org.apache.avro.file.DataFileStream;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumReader;
+import org.apache.commons.lang.StringUtils;
+import org.apache.flume.Channel;
+import org.apache.flume.ChannelException;
+import org.apache.flume.Clock;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.EventDeliveryException;
+import org.apache.flume.Sink.Status;
+import org.apache.flume.SystemClock;
+import org.apache.flume.Transaction;
+import org.apache.flume.channel.BasicTransactionSemantics;
+import org.apache.flume.channel.MemoryChannel;
+import org.apache.flume.conf.Configurables;
+import org.apache.flume.event.EventBuilder;
+import org.apache.flume.event.SimpleEvent;
+import org.apache.flume.instrumentation.SinkCounter;
+import org.apache.flume.lifecycle.LifecycleException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.Lists;
+
+public class TestHDFSEventSink {
+
+ private HDFSEventSink sink;
+ private String testPath;
+ private static final Logger LOG = LoggerFactory
+ .getLogger(HDFSEventSink.class);
+
+ static {
+ System.setProperty("java.security.krb5.realm", "flume");
+ System.setProperty("java.security.krb5.kdc", "blah");
+ }
+
+ private void dirCleanup() {
+ Configuration conf = new Configuration();
+ try {
+ FileSystem fs = FileSystem.get(conf);
+ Path dirPath = new Path(testPath);
+ if (fs.exists(dirPath)) {
+ fs.delete(dirPath, true);
+ }
+ } catch (IOException eIO) {
+ LOG.warn("IO Error in test cleanup", eIO);
+ }
+ }
+
+ // TODO: use System.getProperty("file.separator") instead of hardcoded '/'
+ @Before
+ public void setUp() {
+ LOG.debug("Starting...");
+ /*
+ * FIXME: Use a dynamic path to support concurrent test execution. Also,
+ * beware of the case where this path is used for something or when the
+ * Hadoop config points at file:/// rather than hdfs://. We need to find a
+ * better way of testing HDFS related functionality.
+ */
+ testPath = "file:///tmp/flume-test."
+ + Calendar.getInstance().getTimeInMillis() + "."
+ + Thread.currentThread().getId();
+
+ sink = new HDFSEventSink();
+ sink.setName("HDFSEventSink-" + UUID.randomUUID().toString());
+ dirCleanup();
+ }
+
+ @After
+ public void tearDown() {
+ if (System.getenv("hdfs_keepFiles") == null) dirCleanup();
+ }
+
+ @Test
+ public void testTextBatchAppend() throws Exception {
+ doTestTextBatchAppend(false);
+ }
+
+ @Test
+ public void testTextBatchAppendRawFS() throws Exception {
+ doTestTextBatchAppend(true);
+ }
+
+ public void doTestTextBatchAppend(boolean useRawLocalFileSystem)
+ throws Exception {
+ LOG.debug("Starting...");
+
+ final long rollCount = 10;
+ final long batchSize = 2;
+ final String fileName = "FlumeData";
+ String newPath = testPath + "/singleTextBucket";
+ int totalEvents = 0;
+ int i = 1, j = 1;
+
+ // clear the test directory
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ Path dirPath = new Path(newPath);
+ fs.delete(dirPath, true);
+ fs.mkdirs(dirPath);
+
+ Context context = new Context();
+
+ // context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");
+ context.put("hdfs.path", newPath);
+ context.put("hdfs.filePrefix", fileName);
+ context.put("hdfs.rollCount", String.valueOf(rollCount));
+ context.put("hdfs.rollInterval", "0");
+ context.put("hdfs.rollSize", "0");
+ context.put("hdfs.batchSize", String.valueOf(batchSize));
+ context.put("hdfs.writeFormat", "Text");
+ context.put("hdfs.useRawLocalFileSystem",
+ Boolean.toString(useRawLocalFileSystem));
+ context.put("hdfs.fileType", "DataStream");
+
+ Configurables.configure(sink, context);
+
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, context);
+
+ sink.setChannel(channel);
+ sink.start();
+
+ Calendar eventDate = Calendar.getInstance();
+ List<String> bodies = Lists.newArrayList();
+
+ // push the event batches into channel to roll twice
+ for (i = 1; i <= (rollCount * 10) / batchSize; i++) {
+ Transaction txn = channel.getTransaction();
+ txn.begin();
+ for (j = 1; j <= batchSize; j++) {
+ Event event = new SimpleEvent();
+ eventDate.clear();
+ eventDate.set(2011, i, i, i, 0); // yy mm dd
+ String body = "Test." + i + "." + j;
+ event.setBody(body.getBytes());
+ bodies.add(body);
+ channel.put(event);
+ totalEvents++;
+ }
+ txn.commit();
+ txn.close();
+
+ // execute sink to process the events
+ sink.process();
+ }
+
+ sink.stop();
+
+ // loop through all the files generated and check their contains
+ FileStatus[] dirStat = fs.listStatus(dirPath);
+ Path[] fList = FileUtil.stat2Paths(dirStat);
+
+ // check that the roll happened correctly for the given data
+ long expectedFiles = totalEvents / rollCount;
+ if (totalEvents % rollCount > 0) expectedFiles++;
+ Assert.assertEquals("num files wrong, found: " +
+ Lists.newArrayList(fList), expectedFiles, fList.length);
+ // check the contents of the all files
+ verifyOutputTextFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
+ }
+
+ @Test
+ public void testLifecycle() throws InterruptedException, LifecycleException {
+ LOG.debug("Starting...");
+ Context context = new Context();
+
+ context.put("hdfs.path", testPath);
+ /*
+ * context.put("hdfs.rollInterval", String.class);
+ * context.get("hdfs.rollSize", String.class); context.get("hdfs.rollCount",
+ * String.class);
+ */
+ Configurables.configure(sink, context);
+
+ sink.setChannel(new MemoryChannel());
+
+ sink.start();
+ sink.stop();
+ }
+
+ @Test
+ public void testEmptyChannelResultsInStatusBackoff()
+ throws InterruptedException, LifecycleException, EventDeliveryException {
+ LOG.debug("Starting...");
+ Context context = new Context();
+ Channel channel = new MemoryChannel();
+ context.put("hdfs.path", testPath);
+ context.put("keep-alive", "0");
+ Configurables.configure(sink, context);
+ Configurables.configure(channel, context);
+ sink.setChannel(channel);
+ sink.start();
+ Assert.assertEquals(Status.BACKOFF, sink.process());
+ sink.stop();
+ }
+
+ @Test
+ public void testKerbFileAccess() throws InterruptedException,
+ LifecycleException, EventDeliveryException, IOException {
+ LOG.debug("Starting testKerbFileAccess() ...");
+ final String fileName = "FlumeData";
+ final long rollCount = 5;
+ final long batchSize = 2;
+ String newPath = testPath + "/singleBucket";
+ String kerbConfPrincipal = "user1/localhost@EXAMPLE.COM";
+ String kerbKeytab = "/usr/lib/flume/nonexistkeytabfile";
+
+ //turn security on
+ Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+
+ Context context = new Context();
+ context.put("hdfs.path", newPath);
+ context.put("hdfs.filePrefix", fileName);
+ context.put("hdfs.rollCount", String.valueOf(rollCount));
+ context.put("hdfs.batchSize", String.valueOf(batchSize));
+ context.put("hdfs.kerberosPrincipal", kerbConfPrincipal);
+ context.put("hdfs.kerberosKeytab", kerbKeytab);
+
+ try {
+ Configurables.configure(sink, context);
+ Assert.fail("no exception thrown");
+ } catch (IllegalArgumentException expected) {
+ Assert.assertTrue(expected.getMessage().contains(
+ "Keytab is not a readable file"));
+ } finally {
+ //turn security off
+ conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
+ "simple");
+ UserGroupInformation.setConfiguration(conf);
+ }
+ }
+
+ @Test
+ public void testTextAppend() throws InterruptedException, LifecycleException,
+ EventDeliveryException, IOException {
+
+ LOG.debug("Starting...");
+ final long rollCount = 3;
+ final long batchSize = 2;
+ final String fileName = "FlumeData";
+ String newPath = testPath + "/singleTextBucket";
+ int totalEvents = 0;
+ int i = 1, j = 1;
+
+ // clear the test directory
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ Path dirPath = new Path(newPath);
+ fs.delete(dirPath, true);
+ fs.mkdirs(dirPath);
+
+ Context context = new Context();
+
+ // context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");
+ context.put("hdfs.path", newPath);
+ context.put("hdfs.filePrefix", fileName);
+ context.put("hdfs.rollCount", String.valueOf(rollCount));
+ context.put("hdfs.batchSize", String.valueOf(batchSize));
+ context.put("hdfs.writeFormat", "Text");
+ context.put("hdfs.fileType", "DataStream");
+
+ Configurables.configure(sink, context);
+
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, context);
+
+ sink.setChannel(channel);
+ sink.start();
+
+ Calendar eventDate = Calendar.getInstance();
+ List<String> bodies = Lists.newArrayList();
+
+ // push the event batches into channel
+ for (i = 1; i < 4; i++) {
+ Transaction txn = channel.getTransaction();
+ txn.begin();
+ for (j = 1; j <= batchSize; j++) {
+ Event event = new SimpleEvent();
+ eventDate.clear();
+ eventDate.set(2011, i, i, i, 0); // yy mm dd
+ event.getHeaders().put("timestamp",
+ String.valueOf(eventDate.getTimeInMillis()));
+ event.getHeaders().put("hostname", "Host" + i);
+ String body = "Test." + i + "." + j;
+ event.setBody(body.getBytes());
+ bodies.add(body);
+ channel.put(event);
+ totalEvents++;
+ }
+ txn.commit();
+ txn.close();
+
+ // execute sink to process the events
+ sink.process();
+ }
+
+ sink.stop();
+
+ // loop through all the files generated and check their contains
+ FileStatus[] dirStat = fs.listStatus(dirPath);
+ Path[] fList = FileUtil.stat2Paths(dirStat);
+
+ // check that the roll happened correctly for the given data
+ long expectedFiles = totalEvents / rollCount;
+ if (totalEvents % rollCount > 0) expectedFiles++;
+ Assert.assertEquals("num files wrong, found: " +
+ Lists.newArrayList(fList), expectedFiles, fList.length);
+ verifyOutputTextFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
+ }
+
+ @Test
+ public void testAvroAppend() throws InterruptedException, LifecycleException,
+ EventDeliveryException, IOException {
+
+ LOG.debug("Starting...");
+ final long rollCount = 3;
+ final long batchSize = 2;
+ final String fileName = "FlumeData";
+ String newPath = testPath + "/singleTextBucket";
+ int totalEvents = 0;
+ int i = 1, j = 1;
+
+ // clear the test directory
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ Path dirPath = new Path(newPath);
+ fs.delete(dirPath, true);
+ fs.mkdirs(dirPath);
+
+ Context context = new Context();
+
+ // context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");
+ context.put("hdfs.path", newPath);
+ context.put("hdfs.filePrefix", fileName);
+ context.put("hdfs.rollCount", String.valueOf(rollCount));
+ context.put("hdfs.batchSize", String.valueOf(batchSize));
+ context.put("hdfs.writeFormat", "Text");
+ context.put("hdfs.fileType", "DataStream");
+ context.put("serializer", "AVRO_EVENT");
+
+ Configurables.configure(sink, context);
+
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, context);
+
+ sink.setChannel(channel);
+ sink.start();
+
+ Calendar eventDate = Calendar.getInstance();
+ List<String> bodies = Lists.newArrayList();
+
+ // push the event batches into channel
+ for (i = 1; i < 4; i++) {
+ Transaction txn = channel.getTransaction();
+ txn.begin();
+ for (j = 1; j <= batchSize; j++) {
+ Event event = new SimpleEvent();
+ eventDate.clear();
+ eventDate.set(2011, i, i, i, 0); // yy mm dd
+ event.getHeaders().put("timestamp",
+ String.valueOf(eventDate.getTimeInMillis()));
+ event.getHeaders().put("hostname", "Host" + i);
+ String body = "Test." + i + "." + j;
+ event.setBody(body.getBytes());
+ bodies.add(body);
+ channel.put(event);
+ totalEvents++;
+ }
+ txn.commit();
+ txn.close();
+
+ // execute sink to process the events
+ sink.process();
+ }
+
+ sink.stop();
+
+ // loop through all the files generated and check their contains
+ FileStatus[] dirStat = fs.listStatus(dirPath);
+ Path[] fList = FileUtil.stat2Paths(dirStat);
+
+ // check that the roll happened correctly for the given data
+ long expectedFiles = totalEvents / rollCount;
+ if (totalEvents % rollCount > 0) expectedFiles++;
+ Assert.assertEquals("num files wrong, found: " +
+ Lists.newArrayList(fList), expectedFiles, fList.length);
+ verifyOutputAvroFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
+ }
+
+ @Test
+ public void testSimpleAppend() throws InterruptedException,
+ LifecycleException, EventDeliveryException, IOException {
+
+ LOG.debug("Starting...");
+ final String fileName = "FlumeData";
+ final long rollCount = 5;
+ final long batchSize = 2;
+ final int numBatches = 4;
+ String newPath = testPath + "/singleBucket";
+ int totalEvents = 0;
+ int i = 1, j = 1;
+
+ // clear the test directory
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ Path dirPath = new Path(newPath);
+ fs.delete(dirPath, true);
+ fs.mkdirs(dirPath);
+
+ Context context = new Context();
+
+ context.put("hdfs.path", newPath);
+ context.put("hdfs.filePrefix", fileName);
+ context.put("hdfs.rollCount", String.valueOf(rollCount));
+ context.put("hdfs.batchSize", String.valueOf(batchSize));
+
+ Configurables.configure(sink, context);
+
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, context);
+
+ sink.setChannel(channel);
+ sink.start();
+
+ Calendar eventDate = Calendar.getInstance();
+ List<String> bodies = Lists.newArrayList();
+
+ // push the event batches into channel
+ for (i = 1; i < numBatches; i++) {
+ Transaction txn = channel.getTransaction();
+ txn.begin();
+ for (j = 1; j <= batchSize; j++) {
+ Event event = new SimpleEvent();
+ eventDate.clear();
+ eventDate.set(2011, i, i, i, 0); // yy mm dd
+ event.getHeaders().put("timestamp",
+ String.valueOf(eventDate.getTimeInMillis()));
+ event.getHeaders().put("hostname", "Host" + i);
+ String body = "Test." + i + "." + j;
+ event.setBody(body.getBytes());
+ bodies.add(body);
+ channel.put(event);
+ totalEvents++;
+ }
+ txn.commit();
+ txn.close();
+
+ // execute sink to process the events
+ sink.process();
+ }
+
+ sink.stop();
+
+ // loop through all the files generated and check their contains
+ FileStatus[] dirStat = fs.listStatus(dirPath);
+ Path[] fList = FileUtil.stat2Paths(dirStat);
+
+ // check that the roll happened correctly for the given data
+ long expectedFiles = totalEvents / rollCount;
+ if (totalEvents % rollCount > 0) expectedFiles++;
+ Assert.assertEquals("num files wrong, found: " +
+ Lists.newArrayList(fList), expectedFiles, fList.length);
+ verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
+ }
+
+ @Test
+ public void testSimpleAppendLocalTime()
+ throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
+ final long currentTime = System.currentTimeMillis();
+ Clock clk = new Clock() {
+ @Override
+ public long currentTimeMillis() {
+ return currentTime;
+ }
+ };
+
+ LOG.debug("Starting...");
+ final String fileName = "FlumeData";
+ final long rollCount = 5;
+ final long batchSize = 2;
+ final int numBatches = 4;
+ String newPath = testPath + "/singleBucket/%s" ;
+ String expectedPath = testPath + "/singleBucket/" +
+ String.valueOf(currentTime / 1000);
+ int totalEvents = 0;
+ int i = 1, j = 1;
+
+ // clear the test directory
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ Path dirPath = new Path(expectedPath);
+ fs.delete(dirPath, true);
+ fs.mkdirs(dirPath);
+
+ Context context = new Context();
+
+ context.put("hdfs.path", newPath);
+ context.put("hdfs.filePrefix", fileName);
+ context.put("hdfs.rollCount", String.valueOf(rollCount));
+ context.put("hdfs.batchSize", String.valueOf(batchSize));
+ context.put("hdfs.useLocalTimeStamp", String.valueOf(true));
+
+ Configurables.configure(sink, context);
+
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, context);
+
+ sink.setChannel(channel);
+ sink.setBucketClock(clk);
+ sink.start();
+
+ Calendar eventDate = Calendar.getInstance();
+ List<String> bodies = Lists.newArrayList();
+
+ // push the event batches into channel
+ for (i = 1; i < numBatches; i++) {
+ Transaction txn = channel.getTransaction();
+ txn.begin();
+ for (j = 1; j <= batchSize; j++) {
+ Event event = new SimpleEvent();
+ eventDate.clear();
+ eventDate.set(2011, i, i, i, 0); // yy mm dd
+ event.getHeaders().put("timestamp",
+ String.valueOf(eventDate.getTimeInMillis()));
+ event.getHeaders().put("hostname", "Host" + i);
+ String body = "Test." + i + "." + j;
+ event.setBody(body.getBytes());
+ bodies.add(body);
+ channel.put(event);
+ totalEvents++;
+ }
+ txn.commit();
+ txn.close();
+
+ // execute sink to process the events
+ sink.process();
+ }
+
+ sink.stop();
+
+ // loop through all the files generated and check their contains
+ FileStatus[] dirStat = fs.listStatus(dirPath);
+ Path[] fList = FileUtil.stat2Paths(dirStat);
+
+ // check that the roll happened correctly for the given data
+ long expectedFiles = totalEvents / rollCount;
+ if (totalEvents % rollCount > 0) expectedFiles++;
+ Assert.assertEquals("num files wrong, found: " +
+ Lists.newArrayList(fList), expectedFiles, fList.length);
+ verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
+ // The clock in bucketpath is static, so restore the real clock
+ sink.setBucketClock(new SystemClock());
+ }
+
+ @Test
+ public void testAppend() throws InterruptedException, LifecycleException,
+ EventDeliveryException, IOException {
+
+ LOG.debug("Starting...");
+ final long rollCount = 3;
+ final long batchSize = 2;
+ final String fileName = "FlumeData";
+
+ // clear the test directory
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ Path dirPath = new Path(testPath);
+ fs.delete(dirPath, true);
+ fs.mkdirs(dirPath);
+
+ Context context = new Context();
+
+ context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");
+ context.put("hdfs.timeZone", "UTC");
+ context.put("hdfs.filePrefix", fileName);
+ context.put("hdfs.rollCount", String.valueOf(rollCount));
+ context.put("hdfs.batchSize", String.valueOf(batchSize));
+
+ Configurables.configure(sink, context);
+
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, context);
+
+ sink.setChannel(channel);
+ sink.start();
+
+ Calendar eventDate = Calendar.getInstance();
+ List<String> bodies = Lists.newArrayList();
+ // push the event batches into channel
+ for (int i = 1; i < 4; i++) {
+ Transaction txn = channel.getTransaction();
+ txn.begin();
+ for (int j = 1; j <= batchSize; j++) {
+ Event event = new SimpleEvent();
+ eventDate.clear();
+ eventDate.set(2011, i, i, i, 0); // yy mm dd
+ event.getHeaders().put("timestamp",
+ String.valueOf(eventDate.getTimeInMillis()));
+ event.getHeaders().put("hostname", "Host" + i);
+ String body = "Test." + i + "." + j;
+ event.setBody(body.getBytes());
+ bodies.add(body);
+ channel.put(event);
+ }
+ txn.commit();
+ txn.close();
+
+ // execute sink to process the events
+ sink.process();
+ }
+
+ sink.stop();
+ verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
+ }
+
+ // inject fault and make sure that the txn is rolled back and retried
+ @Test
+ public void testBadSimpleAppend() throws InterruptedException,
+ LifecycleException, EventDeliveryException, IOException {
+
+ LOG.debug("Starting...");
+ final String fileName = "FlumeData";
+ final long rollCount = 5;
+ final long batchSize = 2;
+ final int numBatches = 4;
+ String newPath = testPath + "/singleBucket";
+ int totalEvents = 0;
+ int i = 1, j = 1;
+
+ HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
+ sink = new HDFSEventSink(badWriterFactory);
+
+ // clear the test directory
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ Path dirPath = new Path(newPath);
+ fs.delete(dirPath, true);
+ fs.mkdirs(dirPath);
+
+ Context context = new Context();
+
+ context.put("hdfs.path", newPath);
+ context.put("hdfs.filePrefix", fileName);
+ context.put("hdfs.rollCount", String.valueOf(rollCount));
+ context.put("hdfs.batchSize", String.valueOf(batchSize));
+ context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
+
+ Configurables.configure(sink, context);
+
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, context);
+
+ sink.setChannel(channel);
+ sink.start();
+
+ Calendar eventDate = Calendar.getInstance();
+
+ List<String> bodies = Lists.newArrayList();
+ // push the event batches into channel
+ for (i = 1; i < numBatches; i++) {
+ Transaction txn = channel.getTransaction();
+ txn.begin();
+ for (j = 1; j <= batchSize; j++) {
+ Event event = new SimpleEvent();
+ eventDate.clear();
+ eventDate.set(2011, i, i, i, 0); // yy mm dd
+ event.getHeaders().put("timestamp",
+ String.valueOf(eventDate.getTimeInMillis()));
+ event.getHeaders().put("hostname", "Host" + i);
+
+ String body = "Test." + i + "." + j;
+ event.setBody(body.getBytes());
+ bodies.add(body);
+ // inject fault
+ if ((totalEvents % 30) == 1) {
+ event.getHeaders().put("fault-once", "");
+ }
+ channel.put(event);
+ totalEvents++;
+ }
+ txn.commit();
+ txn.close();
+
+ LOG.info("Process events: " + sink.process());
+ }
+ LOG.info("Process events to end of transaction max: " + sink.process());
+ LOG.info("Process events to injected fault: " + sink.process());
+ LOG.info("Process events remaining events: " + sink.process());
+ sink.stop();
+ verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
+
+ SinkCounter sc = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
+ Assert.assertEquals(1, sc.getEventWriteFail());
+ }
+
+
+ private List<String> getAllFiles(String input) {
+ List<String> output = Lists.newArrayList();
+ File dir = new File(input);
+ if (dir.isFile()) {
+ output.add(dir.getAbsolutePath());
+ } else if (dir.isDirectory()) {
+ for (String file : dir.list()) {
+ File subDir = new File(dir, file);
+ output.addAll(getAllFiles(subDir.getAbsolutePath()));
+ }
+ }
+ return output;
+ }
+
+ private void verifyOutputSequenceFiles(FileSystem fs, Configuration conf, String dir,
+ String prefix, List<String> bodies) throws IOException {
+ int found = 0;
+ int expected = bodies.size();
+ for (String outputFile : getAllFiles(dir)) {
+ String name = (new File(outputFile)).getName();
+ if (name.startsWith(prefix)) {
+ SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(outputFile), conf);
+ LongWritable key = new LongWritable();
+ BytesWritable value = new BytesWritable();
+ while (reader.next(key, value)) {
+ String body = new String(value.getBytes(), 0, value.getLength());
+ if (bodies.contains(body)) {
+ LOG.debug("Found event body: {}", body);
+ bodies.remove(body);
+ found++;
+ }
+ }
+ reader.close();
+ }
+ }
+ if (!bodies.isEmpty()) {
+ for (String body : bodies) {
+ LOG.error("Never found event body: {}", body);
+ }
+ }
+ Assert.assertTrue("Found = " + found + ", Expected = " +
+ expected + ", Left = " + bodies.size() + " " + bodies,
+ bodies.size() == 0);
+
+ }
+
+ private void verifyOutputTextFiles(FileSystem fs, Configuration conf, String dir, String prefix,
+ List<String> bodies) throws IOException {
+ int found = 0;
+ int expected = bodies.size();
+ for (String outputFile : getAllFiles(dir)) {
+ String name = (new File(outputFile)).getName();
+ if (name.startsWith(prefix)) {
+ FSDataInputStream input = fs.open(new Path(outputFile));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(input));
+ String body = null;
+ while ((body = reader.readLine()) != null) {
+ bodies.remove(body);
+ found++;
+ }
+ reader.close();
+ }
+ }
+ Assert.assertTrue("Found = " + found + ", Expected = " +
+ expected + ", Left = " + bodies.size() + " " + bodies,
+ bodies.size() == 0);
+
+ }
+
+ private void verifyOutputAvroFiles(FileSystem fs, Configuration conf, String dir, String prefix,
+ List<String> bodies) throws IOException {
+ int found = 0;
+ int expected = bodies.size();
+ for (String outputFile : getAllFiles(dir)) {
+ String name = (new File(outputFile)).getName();
+ if (name.startsWith(prefix)) {
+ FSDataInputStream input = fs.open(new Path(outputFile));
+ DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
+ DataFileStream<GenericRecord> avroStream =
+ new DataFileStream<GenericRecord>(input, reader);
+ GenericRecord record = new GenericData.Record(avroStream.getSchema());
+ while (avroStream.hasNext()) {
+ avroStream.next(record);
+ ByteBuffer body = (ByteBuffer) record.get("body");
+ CharsetDecoder decoder = Charsets.UTF_8.newDecoder();
+ String bodyStr = decoder.decode(body).toString();
+ LOG.debug("Removing event: {}", bodyStr);
+ bodies.remove(bodyStr);
+ found++;
+ }
+ avroStream.close();
+ input.close();
+ }
+ }
+ Assert.assertTrue("Found = " + found + ", Expected = " +
+ expected + ", Left = " + bodies.size() + " " + bodies,
+ bodies.size() == 0);
+ }
+
+ /**
+ * Ensure that when a write throws an IOException we are
+ * able to continue to progress in the next process() call.
+ * This relies on Transactional rollback semantics for durability and
+ * the behavior of the BucketWriter class of close()ing upon IOException.
+ */
+ @Test
+ public void testCloseReopen()
+ throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
+
+ LOG.debug("Starting...");
+ final int numBatches = 4;
+ final String fileName = "FlumeData";
+ final long rollCount = 5;
+ final long batchSize = 2;
+ String newPath = testPath + "/singleBucket";
+ int i = 1, j = 1;
+
+ HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
+ sink = new HDFSEventSink(badWriterFactory);
+
+ // clear the test directory
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ Path dirPath = new Path(newPath);
+ fs.delete(dirPath, true);
+ fs.mkdirs(dirPath);
+
+ Context context = new Context();
+
+ context.put("hdfs.path", newPath);
+ context.put("hdfs.filePrefix", fileName);
+ context.put("hdfs.rollCount", String.valueOf(rollCount));
+ context.put("hdfs.batchSize", String.valueOf(batchSize));
+ context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
+
+ Configurables.configure(sink, context);
+
+ MemoryChannel channel = new MemoryChannel();
+ Configurables.configure(channel, new Context());
+
+ sink.setChannel(channel);
+ sink.start();
+
+ Calendar eventDate = Calendar.getInstance();
+ List<String> bodies = Lists.newArrayList();
+ // push the event batches into channel
+ for (i = 1; i < numBatches; i++) {
+ channel.getTransaction().begin();
+ try {
+ for (j = 1; j <= batchSize; j++) {
+ Event event = new SimpleEvent();
+ eventDate.clear();
+ eventDate.set(2011, i, i, i, 0); // yy mm dd
+ event.getHeaders().put("timestamp",
+ String.valueOf(eventDate.getTimeInMillis()));
+ event.getHeaders().put("hostname", "Host" + i);
+ String body = "Test." + i + "." + j;
+ event.setBody(body.getBytes());
+ bodies.add(body);
+ // inject fault
+ event.getHeaders().put("fault-until-reopen", "");
+ channel.put(event);
+ }
+ channel.getTransaction().commit();
+ } finally {
+ channel.getTransaction().close();
+ }
+ LOG.info("execute sink to process the events: " + sink.process());
+ }
+ LOG.info("clear any events pending due to errors: " + sink.process());
+ sink.stop();
+
+ verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
+
+ SinkCounter sc = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
+ Assert.assertEquals(1, sc.getEventWriteFail());
+ }
+
+ /**
+ * Test that the old bucket writer is closed at the end of rollInterval and
+ * a new one is used for the next set of events.
+ */
+ @Test
+ public void testCloseReopenOnRollTime()
+ throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
+
+ LOG.debug("Starting...");
+ final int numBatches = 4;
+ final String fileName = "FlumeData";
+ final long batchSize = 2;
+ String newPath = testPath + "/singleBucket";
+ int i = 1, j = 1;
+
+ HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
+ sink = new HDFSEventSink(badWriterFactory);
+
+ // clear the test directory
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ Path dirPath = new Path(newPath);
+ fs.delete(dirPath, true);
+ fs.mkdirs(dirPath);
+
+ Context context = new Context();
+
+ context.put("hdfs.path", newPath);
+ context.put("hdfs.filePrefix", fileName);
+ context.put("hdfs.rollCount", String.valueOf(0));
+ context.put("hdfs.rollSize", String.valueOf(0));
+ context.put("hdfs.rollInterval", String.valueOf(2));
+ context.put("hdfs.batchSize", String.valueOf(batchSize));
+ context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
+
+ Configurables.configure(sink, context);
+
+ MemoryChannel channel = new MemoryChannel();
+ Configurables.configure(channel, new Context());
+
+ sink.setChannel(channel);
+ sink.start();
+
+ Calendar eventDate = Calendar.getInstance();
+ List<String> bodies = Lists.newArrayList();
+ // push the event batches into channel
+ for (i = 1; i < numBatches; i++) {
+ channel.getTransaction().begin();
+ try {
+ for (j = 1; j <= batchSize; j++) {
+ Event event = new SimpleEvent();
+ eventDate.clear();
+ eventDate.set(2011, i, i, i, 0); // yy mm dd
+ event.getHeaders().put("timestamp",
+ String.valueOf(eventDate.getTimeInMillis()));
+ event.getHeaders().put("hostname", "Host" + i);
+ String body = "Test." + i + "." + j;
+ event.setBody(body.getBytes());
+ bodies.add(body);
+ // inject fault
+ event.getHeaders().put("count-check", "");
+ channel.put(event);
+ }
+ channel.getTransaction().commit();
+ } finally {
+ channel.getTransaction().close();
+ }
+ LOG.info("execute sink to process the events: " + sink.process());
+ // Make sure the first file gets rolled due to rollTimeout.
+ if (i == 1) {
+ Thread.sleep(2001);
+ }
+ }
+ LOG.info("clear any events pending due to errors: " + sink.process());
+ sink.stop();
+
+ Assert.assertTrue(badWriterFactory.openCount.get() >= 2);
+ LOG.info("Total number of bucket writers opened: {}",
+ badWriterFactory.openCount.get());
+ verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName,
+ bodies);
+ }
+
+ /**
+ * Test that a close due to roll interval removes the bucketwriter from
+ * sfWriters map.
+ */
+ @Test
+ public void testCloseRemovesFromSFWriters()
+ throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
+
+ LOG.debug("Starting...");
+ final String fileName = "FlumeData";
+ final long batchSize = 2;
+ String newPath = testPath + "/singleBucket";
+ int i = 1, j = 1;
+
+ HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
+ sink = new HDFSEventSink(badWriterFactory);
+
+ // clear the test directory
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ Path dirPath = new Path(newPath);
+ fs.delete(dirPath, true);
+ fs.mkdirs(dirPath);
+
+ Context context = new Context();
+
+ context.put("hdfs.path", newPath);
+ context.put("hdfs.filePrefix", fileName);
+ context.put("hdfs.rollCount", String.valueOf(0));
+ context.put("hdfs.rollSize", String.valueOf(0));
+ context.put("hdfs.rollInterval", String.valueOf(1));
+ context.put("hdfs.batchSize", String.valueOf(batchSize));
+ context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
+ String expectedLookupPath = newPath + "/FlumeData";
+
+ Configurables.configure(sink, context);
+
+ MemoryChannel channel = new MemoryChannel();
+ Configurables.configure(channel, new Context());
+
+ sink.setChannel(channel);
+ sink.start();
+
+ Calendar eventDate = Calendar.getInstance();
+ List<String> bodies = Lists.newArrayList();
+ // push the event batches into channel
+ channel.getTransaction().begin();
+ try {
+ for (j = 1; j <= 2 * batchSize; j++) {
+ Event event = new SimpleEvent();
+ eventDate.clear();
+ eventDate.set(2011, i, i, i, 0); // yy mm dd
+ event.getHeaders().put("timestamp",
+ String.valueOf(eventDate.getTimeInMillis()));
+ event.getHeaders().put("hostname", "Host" + i);
+ String body = "Test." + i + "." + j;
+ event.setBody(body.getBytes());
+ bodies.add(body);
+ // inject fault
+ event.getHeaders().put("count-check", "");
+ channel.put(event);
+ }
+ channel.getTransaction().commit();
+ } finally {
+ channel.getTransaction().close();
+ }
+ LOG.info("execute sink to process the events: " + sink.process());
+ Assert.assertTrue(sink.getSfWriters().containsKey(expectedLookupPath));
+ // Make sure the first file gets rolled due to rollTimeout.
+ Thread.sleep(2001);
+ Assert.assertFalse(sink.getSfWriters().containsKey(expectedLookupPath));
+ LOG.info("execute sink to process the events: " + sink.process());
+ // A new bucket writer should have been created for this bucket. So
+ // sfWriters map should not have the same key again.
+ Assert.assertTrue(sink.getSfWriters().containsKey(expectedLookupPath));
+ sink.stop();
+
+ LOG.info("Total number of bucket writers opened: {}",
+ badWriterFactory.openCount.get());
+ verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName,
+ bodies);
+ }
+
+
+
+ /*
+ * append using slow sink writer.
+ * verify that the process returns backoff due to timeout
+ */
+ @Test
+ public void testSlowAppendFailure() throws InterruptedException,
+ LifecycleException, EventDeliveryException, IOException {
+
+ LOG.debug("Starting...");
+ final String fileName = "FlumeData";
+ final long rollCount = 5;
+ final long batchSize = 2;
+ final int numBatches = 2;
+ String newPath = testPath + "/singleBucket";
+ int i = 1, j = 1;
+
+ // clear the test directory
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ Path dirPath = new Path(newPath);
+ fs.delete(dirPath, true);
+ fs.mkdirs(dirPath);
+
+ // create HDFS sink with slow writer
+ HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();
+ sink = new HDFSEventSink(badWriterFactory);
+
+ Context context = new Context();
+ context.put("hdfs.path", newPath);
+ context.put("hdfs.filePrefix", fileName);
+ context.put("hdfs.rollCount", String.valueOf(rollCount));
+ context.put("hdfs.batchSize", String.valueOf(batchSize));
+ context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
+ context.put("hdfs.callTimeout", Long.toString(1000));
+ Configurables.configure(sink, context);
+
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, context);
+
+ sink.setChannel(channel);
+ sink.start();
+
+ Calendar eventDate = Calendar.getInstance();
+
+ // push the event batches into channel
+ for (i = 0; i < numBatches; i++) {
+ Transaction txn = channel.getTransaction();
+ txn.begin();
+ for (j = 1; j <= batchSize; j++) {
+ Event event = new SimpleEvent();
+ eventDate.clear();
+ eventDate.set(2011, i, i, i, 0); // yy mm dd
+ event.getHeaders().put("timestamp",
+ String.valueOf(eventDate.getTimeInMillis()));
+ event.getHeaders().put("hostname", "Host" + i);
+ event.getHeaders().put("slow", "1500");
+ event.setBody(("Test." + i + "." + j).getBytes());
+ channel.put(event);
+ }
+ txn.commit();
+ txn.close();
+
+ // execute sink to process the events
+ Status satus = sink.process();
+
+ // verify that the append returned backoff due to timeotu
+ Assert.assertEquals(satus, Status.BACKOFF);
+ }
+
+ sink.stop();
+
+ SinkCounter sc = (SinkCounter) Whitebox.getInternalState(sink, "sinkCounter");
+ Assert.assertEquals(2, sc.getEventWriteFail());
+ }
+
+ /*
+ * append using slow sink writer with specified append timeout
+ * verify that the data is written correctly to files
+ */
+ private void slowAppendTestHelper(long appendTimeout)
+ throws InterruptedException, IOException, LifecycleException, EventDeliveryException,
+ IOException {
+ final String fileName = "FlumeData";
+ final long rollCount = 5;
+ final long batchSize = 2;
+ final int numBatches = 2;
+ String newPath = testPath + "/singleBucket";
+ int totalEvents = 0;
+ int i = 1, j = 1;
+
... 9231 lines suppressed ...