You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2023/07/25 04:19:33 UTC

[hudi] branch master updated: [HUDI-3636] Disabling embedded timeline server with spark streaming sink (#9266)

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new e466daf6f4b [HUDI-3636] Disabling embedded timeline server with spark streaming sink (#9266)
e466daf6f4b is described below

commit e466daf6f4b57d5e2069a534b195434cca0e852f
Author: Sivabalan Narayanan <n....@gmail.com>
AuthorDate: Tue Jul 25 00:19:26 2023 -0400

    [HUDI-3636] Disabling embedded timeline server with spark streaming sink (#9266)
---
 .../src/main/scala/org/apache/hudi/HoodieStreamingSink.scala         | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
index 895e8fa5ab7..5667c8870d3 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
@@ -118,6 +118,11 @@ class HoodieStreamingSink(sqlContext: SQLContext,
     // we need auto adjustment enabled for streaming sink since async table services are feasible within the same JVM.
     updatedOptions = updatedOptions.updated(HoodieWriteConfig.AUTO_ADJUST_LOCK_CONFIGS.key, "true")
     updatedOptions = updatedOptions.updated(HoodieSparkSqlWriter.SPARK_STREAMING_BATCH_ID, batchId.toString)
+    if (!options.containsKey(HoodieWriteConfig.EMBEDDED_TIMELINE_SERVER_ENABLE.key())) {
+      // if user does not explicitly override, we are disabling timeline server for streaming sink.
+      // refer to HUDI-3636 for more details
+      updatedOptions = updatedOptions.updated(HoodieWriteConfig.EMBEDDED_TIMELINE_SERVER_ENABLE.key(), " false")
+    }
 
     retry(retryCnt, retryIntervalMs)(
       Try(