You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by sr...@apache.org on 2015/07/30 15:54:15 UTC
[1/3] storm git commit: STORM-951. Storm Hive connector leaking
connections.
Repository: storm
Updated Branches:
refs/heads/master c3a93854f -> 8be4c3c3f
STORM-951. Storm Hive connector leaking connections.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/acb36df7
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/acb36df7
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/acb36df7
Branch: refs/heads/master
Commit: acb36df704d292b514d5c8e8437dc2d1c84a93d8
Parents: 3466d35
Author: Sriharsha Chintalapani <ma...@harsha.io>
Authored: Tue Jul 21 15:42:22 2015 -0700
Committer: Sriharsha Chintalapani <ma...@harsha.io>
Committed: Tue Jul 21 15:42:22 2015 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/storm/hive/bolt/HiveBolt.java | 8 ++++----
.../main/java/org/apache/storm/hive/common/HiveWriter.java | 5 ++---
2 files changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/acb36df7/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/HiveBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/HiveBolt.java b/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/HiveBolt.java
index 4d9f5da..b3defae 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/HiveBolt.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/HiveBolt.java
@@ -106,7 +106,7 @@ public class HiveBolt extends BaseRichBolt {
writer.write(options.getMapper().mapRecord(tuple));
currentBatchSize++;
if(currentBatchSize >= options.getBatchSize()) {
- flushAllWriters();
+ flushAllWriters(true);
currentBatchSize = 0;
}
collector.ack(tuple);
@@ -170,10 +170,10 @@ public class HiveBolt extends BaseRichBolt {
}
}
- private void flushAllWriters()
+ private void flushAllWriters(boolean rollToNext)
throws HiveWriter.CommitFailure, HiveWriter.TxnBatchFailure, HiveWriter.TxnFailure, InterruptedException {
for(HiveWriter writer: allWriters.values()) {
- writer.flush(true);
+ writer.flush(rollToNext);
}
}
@@ -196,7 +196,7 @@ public class HiveBolt extends BaseRichBolt {
private void flushAndCloseWriters() {
try {
- flushAllWriters();
+ flushAllWriters(false);
} catch(Exception e) {
LOG.warn("unable to flush hive writers. ", e);
} finally {
http://git-wip-us.apache.org/repos/asf/storm/blob/acb36df7/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveWriter.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveWriter.java b/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveWriter.java
index 726b8e8..7fc33f2 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveWriter.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveWriter.java
@@ -140,8 +140,7 @@ public class HiveWriter {
if(rollToNext) {
txnBatch = nextTxnBatch(recordWriter);
}
- }
- if(rollToNext) {
+ } else if(rollToNext) {
LOG.debug("Switching to next Txn for {}", endPoint);
txnBatch.beginNextTransaction(); // does not block
}
@@ -247,8 +246,8 @@ public class HiveWriter {
return connection.fetchTransactionBatch(txnsPerBatch, recordWriter); // could block
}
});
- LOG.debug("Acquired {}. Switching to first txn", batch);
batch.beginNextTransaction();
+ LOG.debug("Acquired {}. Switching to first txn", batch);
} catch(TimeoutException e) {
throw new TxnBatchFailure(endPoint, e);
} catch(StreamingException e) {
[3/3] storm git commit: Added STORM-951 to Changelog.
Posted by sr...@apache.org.
Added STORM-951 to Changelog.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8be4c3c3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8be4c3c3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8be4c3c3
Branch: refs/heads/master
Commit: 8be4c3c3f85adb109685b898fc343660798a02e1
Parents: 63fe18c
Author: Sriharsha Chintalapani <ha...@hortonworks.com>
Authored: Thu Jul 30 06:53:34 2015 -0700
Committer: Sriharsha Chintalapani <ha...@hortonworks.com>
Committed: Thu Jul 30 06:53:34 2015 -0700
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8be4c3c3/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6da5d07..2ec09e1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.11.0
+ * STORM-951: Storm Hive connector leaking connections.
* STORM_803: Better CI logs
* STORM-806: use storm.zookeeper.connection.timeout in storm-kafka ZkState when newCurator
* STORM-809: topology.message.timeout.secs should not allow for null or <= 0 values
[2/3] storm git commit: Merge branch 'STORM-951' of
https://github.com/harshach/incubator-storm into STORM-951
Posted by sr...@apache.org.
Merge branch 'STORM-951' of https://github.com/harshach/incubator-storm into STORM-951
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/63fe18c6
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/63fe18c6
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/63fe18c6
Branch: refs/heads/master
Commit: 63fe18c6c169aed9d4ee78de242c8c4fe93e20d6
Parents: c3a9385 acb36df
Author: Sriharsha Chintalapani <ha...@hortonworks.com>
Authored: Thu Jul 30 06:43:06 2015 -0700
Committer: Sriharsha Chintalapani <ha...@hortonworks.com>
Committed: Thu Jul 30 06:43:06 2015 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/storm/hive/bolt/HiveBolt.java | 8 ++++----
.../main/java/org/apache/storm/hive/common/HiveWriter.java | 5 ++---
2 files changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------