You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by tl...@apache.org on 2020/08/11 01:59:51 UTC
[incubator-sdap-nexus] 28/28: start fair scheduling implementation
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a commit to branch SDAP-268
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit 04f2b36b336be2bdae4cd6e03834ff8bb1c849f6
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Mon Aug 10 18:58:57 2020 -0700
start fair scheduling implementation
---
analysis/webservice/webapp.py | 5 ++++-
data-access/nexustiles/config/spark_pools.xml | 0
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/analysis/webservice/webapp.py b/analysis/webservice/webapp.py
index 3ee2f09..c74972d 100644
--- a/analysis/webservice/webapp.py
+++ b/analysis/webservice/webapp.py
@@ -110,7 +110,10 @@ if __name__ == "__main__":
if spark_context is None:
from pyspark.sql import SparkSession
- spark = SparkSession.builder.appName("nexus-analysis").getOrCreate()
+ spark = SparkSession.builder.appName("nexus-analysis")\
+ .config("spark.scheduler.mode", "FAIR")\
+ .config("spark.scheduler.allocation.file", "config/spark_pools.xml"))
+ .getOrCreate()
spark_context = spark.sparkContext
handlers.append((clazzWrapper.path,
diff --git a/data-access/nexustiles/config/spark_pools.xml b/data-access/nexustiles/config/spark_pools.xml
new file mode 100644
index 0000000..e69de29