You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ma...@apache.org on 2017/06/01 21:41:50 UTC
[31/44] metron git commit: METRON-961: Spooky intermittent exceptions
in pcap during sync should be handled and explained closes
apache/incubator-metron#592
METRON-961: Spooky intermittent exceptions in pcap during sync should be handled and explained closes apache/incubator-metron#592
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fee758b3
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fee758b3
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fee758b3
Branch: refs/heads/Metron_0.4.0
Commit: fee758b3fb2277067421a95054c3e328730decda
Parents: ce381f3
Author: cstella <ce...@gmail.com>
Authored: Sat May 20 09:55:42 2017 -0400
Committer: cstella <ce...@gmail.com>
Committed: Sat May 20 09:55:42 2017 -0400
----------------------------------------------------------------------
.../metron/spout/pcap/PartitionHDFSWriter.java | 22 ++++++++++++++------
1 file changed, 16 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/fee758b3/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java
index 86697db..ebd60dc 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java
@@ -65,11 +65,10 @@ public class PartitionHDFSWriter implements AutoCloseable, Serializable {
})
,HDFS(new SyncHandler() {
@Override
- public void sync(FSDataOutputStream outputStream) throws IOException{
-
+ public void sync(FSDataOutputStream outputStream) throws IOException {
outputStream.hflush();
outputStream.hsync();
- ((HdfsDataOutputStream)outputStream).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
+ ((HdfsDataOutputStream) outputStream).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
}
})
,LOCAL(new SyncHandler() {
@@ -91,8 +90,13 @@ public class PartitionHDFSWriter implements AutoCloseable, Serializable {
}
@Override
- public void sync(FSDataOutputStream input) throws IOException {
- func.sync(input);
+ public void sync(FSDataOutputStream input) {
+ try {
+ func.sync(input);
+ }
+ catch(IOException ioe) {
+ LOG.warn("Problems during sync, but this shouldn't be too concerning as long as it's intermittent: " + ioe.getMessage(), ioe);
+ }
}
}
@@ -156,7 +160,13 @@ public class PartitionHDFSWriter implements AutoCloseable, Serializable {
public void handle(long ts, byte[] value) throws IOException {
turnoverIfNecessary(ts);
BytesWritable bw = new BytesWritable(value);
- writer.append(new LongWritable(ts), bw);
+ try {
+ writer.append(new LongWritable(ts), bw);
+ }
+ catch(ArrayIndexOutOfBoundsException aioobe) {
+ LOG.warn("This appears to be HDFS-7765 (https://issues.apache.org/jira/browse/HDFS-7765), " +
+ "which is an issue with syncing and not problematic: " + aioobe.getMessage(), aioobe);
+ }
numWritten++;
if(numWritten % config.getSyncEvery() == 0) {
syncHandler.sync(outputStream);