[GitHub] [spark] jkbradley commented on a change in pull request #24705: [SPARK-22340][PYTHON] Save localProperties in thread.local

[GitBox <gi...@apache.org>]

jkbradley commented on a change in pull request #24705: [SPARK-22340][PYTHON] Save localProperties in thread.local
URL: https://github.com/apache/spark/pull/24705#discussion_r287539792
 
 

 ##########
 File path: python/pyspark/tests/test_rdd.py
 ##########
 @@ -761,6 +761,66 @@ def fail_last(x):
         for i in range(4):
             self.assertEqual(i, next(it))
 
+    def test_multiple_group_jobs(self):
+        # SPARK-22340 test setJobGroup for multiple job groups
+
+        import threading
+        import time
+
+        group_A_name = "group_A"
+        group_B_name = "group_B"
+
+        def map_func(x):
+            time.sleep(x)
+            return x
+
+        num_threads = 4
+        thread_list = []
+        # an array which record whether job is cancelled.
+        # the index of the array is the thread index which job run in.
+        is_job_cancelled = [False for x in range(num_threads)]
+
+        def run_job(job_group, index):
+            try:
+                self.sc.setJobGroup(job_group, "test rdd collect with setting job group")
+                result = self.sc.parallelize([3]).map(map_func).collect()
+                is_job_cancelled[index] = False
+                return result
+            except Exception as e:
+                is_job_cancelled[index] = True
+                print("{} with message: {}".format(type(e).__name__, str(e)))
+                return None
+
+        def launch_job_thread(job_group, index):
+            thread = threading.Thread(target=run_job, args=(job_group, index))
+            thread.start()
+            return thread
+
+        # test job succeeded when not cancelled.
+        run_job(group_A_name, 0)
+        self.assertFalse(is_job_cancelled[0], "job didn't succeeded.")
+
+        for i in range(num_threads):
 
 Review comment:
   I'd explain what this is testing in a comment.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org