[beam] branch spark-runner_structured-streaming updated: Limit the number of partitions to make tests go 300% faster

[ie...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

iemejia pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/spark-runner_structured-streaming by this push:
     new 241453b  Limit the number of partitions to make tests go 300% faster
241453b is described below

commit 241453bda3111d472ed698cf776c963f3216d14f
Author: Ismaël Mejía <ie...@gmail.com>
AuthorDate: Fri May 3 22:15:36 2019 +0200

    Limit the number of partitions to make tests go 300% faster
---
 runners/spark/build.gradle | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runners/spark/build.gradle b/runners/spark/build.gradle
index 8ea30ec..d1e657f 100644
--- a/runners/spark/build.gradle
+++ b/runners/spark/build.gradle
@@ -37,6 +37,7 @@ configurations {
 
 test {
   systemProperty "beam.spark.test.reuseSparkContext", "true"
+  systemProperty "spark.sql.shuffle.partitions", "4"
   systemProperty "spark.ui.enabled", "false"
   systemProperty "spark.ui.showConsoleProgress", "false"
   systemProperty "beamTestPipelineOptions", """[
@@ -165,7 +166,7 @@ task validatesStructuredStreamingRunnerBatch(type: Test) {
           "--streaming=false",
   ])
   systemProperty "beamTestPipelineOptions", pipelineOptions
-  systemProperty "beam.spark.test.reuseSparkContext", "true"
+  systemProperty "spark.sql.shuffle.partitions", "4"
   systemProperty "spark.ui.enabled", "false"
   systemProperty "spark.ui.showConsoleProgress", "false"