[GitHub] [incubator-kyuubi] yaooqinn commented on a change in pull request #1743: Fix parallelism of DataGenerator and other enhancements

[GitBox <gi...@apache.org>]

yaooqinn commented on a change in pull request #1743:
URL: https://github.com/apache/incubator-kyuubi/pull/1743#discussion_r783572627



##########
File path: dev/kyuubi-tpcds/src/main/scala/org/apache/kyuubi/tpcds/DataGenerator.scala
##########
@@ -597,52 +594,50 @@ object DataGenerator {
       promotion)
   }
 
-  def run(config: RunConfig): Unit = {
-    val conf = new SparkConf()
-    val db = conf.get(DATABASE_KEY, config.db)
-    val scaleFactor = conf.get(DataGenerator.SCALE_FACTOR_KEY, config.scaleFactor.toString).toInt
-
+  def run(config: Config): Unit = {
     val spark = SparkSession.builder()
-      .appName(s"Kyuubi TPCDS Generation - ${scaleFactor}GB")
-      .config(conf)
+      .appName(s"Kyuubi TPC-DS Generation - ${config.scaleFactor}GB")
       .enableHiveSupport()
       .getOrCreate()
 
-    logger.info(s"Generating TPCDS tables under database $db")
-    spark.sql(s"CREATE DATABASE IF NOT EXISTS $db")
-    spark.sql(s"USE $db")
-    spark.sql(s"DESC DATABASE $db").show()
-
-    val parallel = spark.sparkContext.defaultParallelism
+    logger.info(s"Generating TPC-DS tables under database ${config.db}")
+    spark.sql(s"CREATE DATABASE IF NOT EXISTS ${config.db}")
+    spark.sql(s"USE ${config.db}")
+    spark.sql(s"DESC DATABASE ${config.db}").show()
 
     val tpcdsTables = initTable(spark)
     tpcdsTables.par.foreach { table =>
-      table.setScaleFactor(scaleFactor)
-      table.setParallelism(parallel)
+      table.setScaleFactor(config.scaleFactor)
+      table.setFormat(config.format)
+      config.parallel.foreach(table.setParallelism)
       spark.sparkContext.setJobDescription(table.toString)
-      logger.info(s"$table")
+      logger.info(s"Generating $table")
       table.create()
     }
   }
 
   def main(args: Array[String]): Unit = {
-    val parser = new scopt.OptionParser[RunConfig]("tpcds-data-generator") {
-      head("tpcds-data-generator", "")
+    val parser = new scopt.OptionParser[Config]("tpcds-data-generator") {
+      head("Kyuubi TPC-DS Data Generator")
       opt[String]('d', "db")
         .action { (x, c) => c.copy(db = x) }
         .text("the databases to write data")

Review comment:
       the database to write data




-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@kyuubi.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org