You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ki...@apache.org on 2022/08/31 05:15:03 UTC
[incubator-seatunnel] branch dev updated: Fix spark streaming batch interval configuration parameter (#2575)
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 6fcc5341a Fix spark streaming batch interval configuration parameter (#2575)
6fcc5341a is described below
commit 6fcc5341a9ecfc1a73877e3ea95807d230f23c7d
Author: miaoze8 <st...@qq.com>
AuthorDate: Wed Aug 31 13:14:56 2022 +0800
Fix spark streaming batch interval configuration parameter (#2575)
Co-authored-by: ze.miao <ze...@pcitc.com>
---
config/spark.streaming.conf.template | 2 +-
deploy/docker/spark/config/application.conf | 2 +-
docs/zh-CN/spark/quick-start.md | 2 +-
.../src/test/java/org/apache/seatunnel/config/ConfigFactoryTest.java | 4 ++--
.../seatunnel-config-shade/src/test/resources/factory/config.conf | 2 +-
.../src/test/resources/seatunnel/variables.conf | 2 +-
.../seatunnel-core-spark/src/test/resources/spark_application.conf | 2 +-
.../seatunnel-spark-starter/src/test/resources/spark_application.conf | 2 +-
8 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/config/spark.streaming.conf.template b/config/spark.streaming.conf.template
index b79a9dec6..2b09071b6 100644
--- a/config/spark.streaming.conf.template
+++ b/config/spark.streaming.conf.template
@@ -26,7 +26,7 @@ env {
spark.executor.instances = 2
spark.executor.cores = 1
spark.executor.memory = "1g"
- spark.streaming.batchDuration = 5
+ spark.stream.batchDuration = 5
}
source {
diff --git a/deploy/docker/spark/config/application.conf b/deploy/docker/spark/config/application.conf
index 1c324ef1a..cf795ae08 100644
--- a/deploy/docker/spark/config/application.conf
+++ b/deploy/docker/spark/config/application.conf
@@ -16,7 +16,7 @@
#
env {
- spark.streaming.batchDuration = 5
+ spark.stream.batchDuration = 5
spark.app.name = "seatunnel"
spark.ui.port = 13000
}
diff --git a/docs/zh-CN/spark/quick-start.md b/docs/zh-CN/spark/quick-start.md
index de3e98bfd..24c017f42 100644
--- a/docs/zh-CN/spark/quick-start.md
+++ b/docs/zh-CN/spark/quick-start.md
@@ -34,7 +34,7 @@ ln -s seatunnel-<version> seatunnel
```bash
env {
# seatunnel defined streaming batch duration in seconds
- spark.streaming.batchDuration = 5
+ spark.stream.batchDuration = 5
spark.app.name = "seatunnel"
spark.ui.port = 13000
diff --git a/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/ConfigFactoryTest.java b/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/ConfigFactoryTest.java
index 016a2f517..4f24ab7af 100644
--- a/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/ConfigFactoryTest.java
+++ b/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/ConfigFactoryTest.java
@@ -47,7 +47,7 @@ public class ConfigFactoryTest {
Assertions.assertEquals("2", env.getString("spark.executor.instances"));
Assertions.assertEquals("1", env.getString("spark.executor.cores"));
Assertions.assertEquals("1g", env.getString("spark.executor.memory"));
- Assertions.assertEquals("5", env.getString("spark.streaming.batchDuration"));
+ Assertions.assertEquals("5", env.getString("spark.stream.batchDuration"));
// check custom plugin
Assertions.assertEquals("c.Console", config.getConfigList("sink").get(1).getString("plugin_name"));
@@ -73,7 +73,7 @@ public class ConfigFactoryTest {
@Test
public void testQuotedString() throws URISyntaxException {
List<String> keys = Arrays.asList("spark.app.name", "spark.executor.instances", "spark.executor.cores",
- "spark.executor.memory", "spark.streaming.batchDuration");
+ "spark.executor.memory", "spark.stream.batchDuration");
Config config = ConfigFactory.parseFile(FileUtils.getFileFromResources("/factory/config.conf"));
Config evnConfig = config.getConfig("env");
diff --git a/seatunnel-config/seatunnel-config-shade/src/test/resources/factory/config.conf b/seatunnel-config/seatunnel-config-shade/src/test/resources/factory/config.conf
index 15e497ebc..af8c82b8f 100644
--- a/seatunnel-config/seatunnel-config-shade/src/test/resources/factory/config.conf
+++ b/seatunnel-config/seatunnel-config-shade/src/test/resources/factory/config.conf
@@ -24,7 +24,7 @@ env {
spark.executor.instances = 2
"spark.executor.cores" = 1
"spark.executor.memory" = "1g"
- "spark.streaming.batchDuration" = 5
+ "spark.stream.batchDuration" = 5
}
source {
diff --git a/seatunnel-config/seatunnel-config-shade/src/test/resources/seatunnel/variables.conf b/seatunnel-config/seatunnel-config-shade/src/test/resources/seatunnel/variables.conf
index fb7983fb8..f06fb1c16 100644
--- a/seatunnel-config/seatunnel-config-shade/src/test/resources/seatunnel/variables.conf
+++ b/seatunnel-config/seatunnel-config-shade/src/test/resources/seatunnel/variables.conf
@@ -14,7 +14,7 @@
# limitations under the License.
spark {
- spark.streaming.batchDuration = 5
+ spark.stream.batchDuration = 5
spark.app.name = "SeaTunnel"
spark.executor.instances = 2
diff --git a/seatunnel-core/seatunnel-core-spark/src/test/resources/spark_application.conf b/seatunnel-core/seatunnel-core-spark/src/test/resources/spark_application.conf
index bf853a4c2..41cf9d55e 100644
--- a/seatunnel-core/seatunnel-core-spark/src/test/resources/spark_application.conf
+++ b/seatunnel-core/seatunnel-core-spark/src/test/resources/spark_application.conf
@@ -24,7 +24,7 @@ env {
spark.executor.instances = 2
"spark.executor.cores" = 1
"spark.executor.memory" = "1g"
- "spark.streaming.batchDuration" = 5
+ "spark.stream.batchDuration" = 5
}
source {
diff --git a/seatunnel-core/seatunnel-spark-starter/src/test/resources/spark_application.conf b/seatunnel-core/seatunnel-spark-starter/src/test/resources/spark_application.conf
index bf853a4c2..41cf9d55e 100644
--- a/seatunnel-core/seatunnel-spark-starter/src/test/resources/spark_application.conf
+++ b/seatunnel-core/seatunnel-spark-starter/src/test/resources/spark_application.conf
@@ -24,7 +24,7 @@ env {
spark.executor.instances = 2
"spark.executor.cores" = 1
"spark.executor.memory" = "1g"
- "spark.streaming.batchDuration" = 5
+ "spark.stream.batchDuration" = 5
}
source {