You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2018/12/02 07:16:30 UTC
[1/3] storm git commit: STORM-3292: flush writers in HiveState when
the trident batch commits
Repository: storm
Updated Branches:
refs/heads/master 6d871a73e -> 94cd157c0
STORM-3292: flush writers in HiveState when the trident batch commits
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/07634fcb
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/07634fcb
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/07634fcb
Branch: refs/heads/master
Commit: 07634fcb881a53d548827f27b1a9388ff8982f5e
Parents: 29eb449
Author: Arun Mahadevan <ar...@apache.org>
Authored: Mon Nov 26 18:15:58 2018 -0800
Committer: Arun Mahadevan <ar...@apache.org>
Committed: Mon Nov 26 18:17:55 2018 -0800
----------------------------------------------------------------------
.../main/java/org/apache/storm/hive/trident/HiveState.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/07634fcb/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveState.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveState.java b/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveState.java
index 6717329..b9494da 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveState.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveState.java
@@ -64,6 +64,13 @@ public class HiveState implements State {
@Override
public void commit(Long txId) {
+ try {
+ flushAllWriters();
+ currentBatchSize = 0;
+ } catch (HiveWriter.TxnFailure | InterruptedException | HiveWriter.CommitFailure | HiveWriter.TxnBatchFailure ex) {
+ LOG.warn("Commit failed. Failing the batch.", ex);
+ throw new FailedException(ex);
+ }
}
public void prepare(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
[2/3] storm git commit: STORM-3292: log an error message if tick
tuple interval is set in HiveOptions for Trident
Posted by ka...@apache.org.
STORM-3292: log an error message if tick tuple interval is set in HiveOptions for Trident
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b7a8d749
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b7a8d749
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b7a8d749
Branch: refs/heads/master
Commit: b7a8d7492c9036fbf5e5c269f121265e68c65cd0
Parents: 07634fc
Author: Arun Mahadevan <ar...@apache.org>
Authored: Tue Nov 27 10:30:28 2018 -0800
Committer: Arun Mahadevan <ar...@apache.org>
Committed: Tue Nov 27 11:00:36 2018 -0800
----------------------------------------------------------------------
.../main/java/org/apache/storm/hive/common/HiveOptions.java | 2 +-
.../java/org/apache/storm/hive/trident/HiveStateFactory.java | 7 +++++++
2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b7a8d749/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveOptions.java b/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveOptions.java
index 4a91da1..db0fa69 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveOptions.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveOptions.java
@@ -19,7 +19,7 @@ public class HiveOptions implements Serializable {
/**
* Half of the default Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS
*/
- private static final int DEFAULT_TICK_TUPLE_INTERVAL_SECS = 15;
+ public static final int DEFAULT_TICK_TUPLE_INTERVAL_SECS = 15;
protected HiveMapper mapper;
protected String databaseName;
http://git-wip-us.apache.org/repos/asf/storm/blob/b7a8d749/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveStateFactory.java b/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveStateFactory.java
index 68722d1..e6a7d84 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveStateFactory.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveStateFactory.java
@@ -27,7 +27,14 @@ public class HiveStateFactory implements StateFactory {
public HiveStateFactory() {}
+ /**
+ * The options for connecting to Hive.
+ */
public HiveStateFactory withOptions(HiveOptions options) {
+ if (options.getTickTupleInterval() != HiveOptions.DEFAULT_TICK_TUPLE_INTERVAL_SECS) {
+ LOG.error("Tick tuple interval will be ignored for trident."
+ + " The Hive writers are flushed after each batch.");
+ }
this.options = options;
return this;
}
[3/3] storm git commit: Merge branch 'STORM-3292' of
https://github.com/arunmahadevan/storm into STORM-3292
Posted by ka...@apache.org.
Merge branch 'STORM-3292' of https://github.com/arunmahadevan/storm into STORM-3292
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/94cd157c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/94cd157c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/94cd157c
Branch: refs/heads/master
Commit: 94cd157c0f12ceb6538169ce4b4f6193f7f77eed
Parents: 6d871a7 b7a8d74
Author: Jungtaek Lim (HeartSaVioR) <ka...@gmail.com>
Authored: Sun Dec 2 15:52:56 2018 +0900
Committer: Jungtaek Lim (HeartSaVioR) <ka...@gmail.com>
Committed: Sun Dec 2 15:52:56 2018 +0900
----------------------------------------------------------------------
.../main/java/org/apache/storm/hive/common/HiveOptions.java | 2 +-
.../main/java/org/apache/storm/hive/trident/HiveState.java | 7 +++++++
.../java/org/apache/storm/hive/trident/HiveStateFactory.java | 7 +++++++
3 files changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------