[jira] [Created] (SPARK-31281) Hit OOM Error - GC Limit

["HongJin (Jira)" <ji...@apache.org>]

HongJin created SPARK-31281:
-------------------------------

             Summary: Hit OOM Error - GC Limit
                 Key: SPARK-31281
                 URL: https://issues.apache.org/jira/browse/SPARK-31281
             Project: Spark
          Issue Type: Question
          Components: Java API
    Affects Versions: 2.4.4
            Reporter: HongJin


conf = new SparkConf().setAppName("test")
 //.set("spark.sql.codegen.wholeStage", "false")
 .set("spark.driver.host", "localhost")
 .set("spark.driver.memory", "4g")
 .set("spark.executor.cores","1")
 .set("spark.num.executors","1")
 .set("spark.executor.memory", "4g")
 .set("spark.executor.memoryOverhead", "400m")
 .set("spark.dynamicAllocation.enabled", "true")
 .set("spark.dynamicAllocation.minExecutors","1")
 .set("spark.dynamicAllocation.maxExecutors","2")
 .set("spark.ui.enabled","true") //enable spark UI
 .set("spark.sql.shuffle.partitions",defaultPartitions)
 .setMaster(numCores)
sparkSession = SparkSession.builder.config(conf).getOrCreate()

 

val df = SparkFactory.sparkSession.sqlContext
 .read
 .option("header", "true")
 .option("delimiter", delimiter)
 .csv(textFileLocation)

 

joinedDf = upperCaseLeft.as("l")
 .join(upperCaseRight.as("r"), caseTransformedKeys, "full_outer")
 .select(compositeKeysCol ::: nonKeyCols.map(col => mapHelper(col,toleranceValue,caseSensitive)): _*)

 

data = joinedDf.take(maxRecords)

 

 

 

 



--
This message was sent by Atlassian Jira
(v8.3.4#803005)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org