You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2023/07/25 04:19:33 UTC
[hudi] branch master updated: [HUDI-3636] Disabling embedded timeline server with spark streaming sink (#9266)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new e466daf6f4b [HUDI-3636] Disabling embedded timeline server with spark streaming sink (#9266)
e466daf6f4b is described below
commit e466daf6f4b57d5e2069a534b195434cca0e852f
Author: Sivabalan Narayanan <n....@gmail.com>
AuthorDate: Tue Jul 25 00:19:26 2023 -0400
[HUDI-3636] Disabling embedded timeline server with spark streaming sink (#9266)
---
.../src/main/scala/org/apache/hudi/HoodieStreamingSink.scala | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
index 895e8fa5ab7..5667c8870d3 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
@@ -118,6 +118,11 @@ class HoodieStreamingSink(sqlContext: SQLContext,
// we need auto adjustment enabled for streaming sink since async table services are feasible within the same JVM.
updatedOptions = updatedOptions.updated(HoodieWriteConfig.AUTO_ADJUST_LOCK_CONFIGS.key, "true")
updatedOptions = updatedOptions.updated(HoodieSparkSqlWriter.SPARK_STREAMING_BATCH_ID, batchId.toString)
+ if (!options.containsKey(HoodieWriteConfig.EMBEDDED_TIMELINE_SERVER_ENABLE.key())) {
+ // if user does not explicitly override, we are disabling timeline server for streaming sink.
+ // refer to HUDI-3636 for more details
+ updatedOptions = updatedOptions.updated(HoodieWriteConfig.EMBEDDED_TIMELINE_SERVER_ENABLE.key(), " false")
+ }
retry(retryCnt, retryIntervalMs)(
Try(