You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2022/05/29 11:41:33 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #2553] Kyuubi Spark TPC-DS Connector - Add tiny scale
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 8578bcd46 [KYUUBI #2553] Kyuubi Spark TPC-DS Connector - Add tiny scale
8578bcd46 is described below
commit 8578bcd46112a790fa7a977a40cd198dbfeb0eee
Author: sychen <sy...@ctrip.com>
AuthorDate: Sun May 29 19:41:19 2022 +0800
[KYUUBI #2553] Kyuubi Spark TPC-DS Connector - Add tiny scale
### _Why are the changes needed?_
Current available scales: 0, 1, 10, 100, 300, 1000, 3000, 10000, 30000, 100000.
Event scale 1G is a little bit large for demo on laptop, we can introduce a scale 0.01 and name the database as `tiny` for demo purpose.
At the same time, it can also shorten the test time of UT.
close #2553
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [x] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
co-author: sychen sychenctrip.com
co-author: Cheng Pan chengpanapache.org
Closes #2774 from pan3793/PR_2773_string.
Closes #2553
0ce9c135 [Cheng Pan] Revert tpcds.tiny stats
3a8febe3 [Cheng Pan] nit
1d310e1e [Cheng Pan] nit
2e087011 [Cheng Pan] nit
21cb411a [Cheng Pan] nit
7db2159b [Cheng Pan] nit
76b40a79 [Cheng Pan] Represent scale in string
fb62f320 [sychen] add tiny scale
Lead-authored-by: sychen <sy...@ctrip.com>
Co-authored-by: Cheng Pan <ch...@apache.org>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.../spark/connector/tpcds/TPCDSBatchScan.scala | 9 +-
.../spark/connector/tpcds/TPCDSCatalog.scala | 9 +-
.../spark/connector/tpcds/TPCDSSchemaUtils.scala | 20 ++++
.../connector/tpcds/TPCDSStatisticsUtils.scala | 131 +++++++++++----------
.../kyuubi/spark/connector/tpcds/TPCDSTable.scala | 4 +-
.../spark/connector/tpcds/TPCDSCatalogSuite.scala | 52 ++++----
.../connector/tpcds/TPCDSSchemaUtilsSuite.scala | 32 +++++
.../spark/kyuubi/TPCDSTableGenerateBenchmark.scala | 2 +-
8 files changed, 159 insertions(+), 100 deletions(-)
diff --git a/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSBatchScan.scala b/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSBatchScan.scala
index 628ef9358..577273fe7 100644
--- a/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSBatchScan.scala
+++ b/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSBatchScan.scala
@@ -30,12 +30,12 @@ import org.apache.spark.sql.connector.read._
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
-case class TPCDSTableChuck(table: String, scale: Int, parallelism: Int, index: Int)
+case class TPCDSTableChuck(table: String, scale: Double, parallelism: Int, index: Int)
extends InputPartition
class TPCDSBatchScan(
@transient table: Table,
- scale: Int,
+ scale: Double,
schema: StructType) extends ScanBuilder
with SupportsReportStatistics with Batch with Serializable {
@@ -55,7 +55,8 @@ class TPCDSBatchScan(
override def toBatch: Batch = this
override def description: String =
- s"Scan TPC-DS sf$scale.${table.getName}, count: ${_numRows}, parallelism: $parallelism"
+ s"Scan TPC-DS ${TPCDSSchemaUtils.dbName(scale)}.${table.getName}, " +
+ s"count: ${_numRows}, parallelism: $parallelism"
override def readSchema: StructType = schema
@@ -75,7 +76,7 @@ class TPCDSBatchScan(
class TPCDSPartitionReader(
table: String,
- scale: Int,
+ scale: Double,
parallelism: Int,
index: Int,
schema: StructType) extends PartitionReader[InternalRow] {
diff --git a/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSCatalog.scala b/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSCatalog.scala
index 5b8933bcd..fd40759d0 100644
--- a/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSCatalog.scala
+++ b/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSCatalog.scala
@@ -29,11 +29,9 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
class TPCDSCatalog extends TableCatalog with SupportsNamespaces {
- val tables: Array[String] = TPCDSSchemaUtils.BASE_TABLES.map(_.getName)
-
- val scales: Array[Int] = TPCDSStatisticsUtils.SCALES
+ val databases: Array[String] = TPCDSSchemaUtils.DATABASES
- val databases: Array[String] = scales.map("sf" + _)
+ val tables: Array[String] = TPCDSSchemaUtils.BASE_TABLES.map(_.getName)
var options: CaseInsensitiveStringMap = _
@@ -53,7 +51,8 @@ class TPCDSCatalog extends TableCatalog with SupportsNamespaces {
override def loadTable(ident: Identifier): SparkTable = (ident.namespace, ident.name) match {
case (Array(db), table) if (databases contains db) && tables.contains(table.toLowerCase) =>
- new TPCDSTable(table.toLowerCase, scales(databases indexOf db), options)
+ val scale = TPCDSSchemaUtils.scale(db)
+ new TPCDSTable(table.toLowerCase, scale, options)
case (_, _) => throw new NoSuchTableException(ident)
}
diff --git a/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSSchemaUtils.scala b/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSSchemaUtils.scala
index beaf528bd..5c34078f0 100644
--- a/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSSchemaUtils.scala
+++ b/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSSchemaUtils.scala
@@ -17,6 +17,8 @@
package org.apache.kyuubi.spark.connector.tpcds
+import java.text.DecimalFormat
+
import scala.collection.JavaConverters._
import io.trino.tpcds.Table
@@ -26,6 +28,24 @@ import io.trino.tpcds.generator._
object TPCDSSchemaUtils {
+ val TINY_SCALE = "0.01"
+
+ val SCALES: Array[String] =
+ Array("0", TINY_SCALE, "1", "10", "100", "300", "1000", "3000", "10000", "30000", "100000")
+
+ val TINY_DB_NAME = "tiny"
+
+ val DATABASES: Array[String] = SCALES.map {
+ case TINY_SCALE => TINY_DB_NAME
+ case scale => s"sf$scale"
+ }
+
+ def normalize(scale: Double): String = new DecimalFormat("#.##").format(scale)
+
+ def scale(dbName: String): Double = SCALES(DATABASES.indexOf(dbName)).toDouble
+
+ def dbName(scale: Double): String = DATABASES(SCALES.indexOf(normalize(scale)))
+
val BASE_TABLES: Array[Table] = Table.getBaseTables.asScala
.filterNot(_.getName == "dbgen_version").toArray
diff --git a/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSStatisticsUtils.scala b/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSStatisticsUtils.scala
index b21b9b4e5..1bcd07bf9 100644
--- a/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSStatisticsUtils.scala
+++ b/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSStatisticsUtils.scala
@@ -20,76 +20,83 @@ package org.apache.kyuubi.spark.connector.tpcds
import io.trino.tpcds.{Scaling, Table}
import io.trino.tpcds.Table._
+import org.apache.kyuubi.spark.connector.tpcds.TPCDSSchemaUtils._
+
// https://tpc.org/TPC_Documents_Current_Versions/pdf/TPC-DS_v3.2.0.pdf
// Page 42 Table 3-2 Database Row Counts
object TPCDSStatisticsUtils {
- val SCALES: Array[Int] = Array(0, 1, 10, 100, 300, 1000, 3000, 10000, 30000, 100000)
-
// https://github.com/Teradata/tpcds/issues/26
- def numRows(table: Table, scale: Int): Long = {
- require(SCALES.contains(scale), s"Unsupported scale $scale")
- (table, scale) match {
- case (_, 0) => 0L
- case (CATALOG_RETURNS, 1) => 144067L
- case (CATALOG_RETURNS, 10) => 1439749L
- case (CATALOG_RETURNS, 100) => 14404374L
- case (CATALOG_RETURNS, 300) => 43193472L
- case (CATALOG_RETURNS, 1000) => 143996756L
- case (CATALOG_RETURNS, 3000) => 432018033L
- case (CATALOG_RETURNS, 10000) => 1440033112L
- case (CATALOG_RETURNS, 30000) => 4319925093L
- case (CATALOG_RETURNS, 100000) => 14400175879L
- case (CATALOG_SALES, 1) => 1441548L
- case (CATALOG_SALES, 10) => 14401261L
- case (CATALOG_SALES, 100) => 143997065L
- case (CATALOG_SALES, 300) => 431969836L
- case (CATALOG_SALES, 1000) => 1439980416L
- case (CATALOG_SALES, 3000) => 4320078880L
- case (CATALOG_SALES, 10000) => 14399964710L
- case (CATALOG_SALES, 30000) => 43200404822L
- case (CATALOG_SALES, 100000) => 143999334399L
- case (STORE_RETURNS, 1) => 287514L
- case (STORE_RETURNS, 10) => 2875432L
- case (STORE_RETURNS, 100) => 28795080L
- case (STORE_RETURNS, 300) => 86393244L
- case (STORE_RETURNS, 1000) => 287999764L
- case (STORE_RETURNS, 3000) => 863989652L
- case (STORE_RETURNS, 10000) => 2879970104L
- case (STORE_RETURNS, 30000) => 8639952111L
- case (STORE_RETURNS, 100000) => 28800018820L
- case (STORE_SALES, 1) => 2880404L
- case (STORE_SALES, 10) => 28800991L
- case (STORE_SALES, 100) => 287997024L
- case (STORE_SALES, 300) => 864001869L
- case (STORE_SALES, 1000) => 2879987999L
- case (STORE_SALES, 3000) => 8639936081L
- case (STORE_SALES, 10000) => 28799983563L
- case (STORE_SALES, 30000) => 86399341874L
- case (STORE_SALES, 100000) => 287997818084L
- case (WEB_RETURNS, 1) => 71763L
- case (WEB_RETURNS, 10) => 719217L
- case (WEB_RETURNS, 100) => 7197670L
- case (WEB_RETURNS, 300) => 21599377L
- case (WEB_RETURNS, 1000) => 71997522L
- case (WEB_RETURNS, 3000) => 216003761L
- case (WEB_RETURNS, 10000) => 720020485L
- case (WEB_RETURNS, 30000) => 2160007345L
- case (WEB_RETURNS, 100000) => 7199904459L
- case (WEB_SALES, 1) => 719384L
- case (WEB_SALES, 10) => 7197566L
- case (WEB_SALES, 100) => 72001237L
- case (WEB_SALES, 300) => 216009853L
- case (WEB_SALES, 1000) => 720000376L
- case (WEB_SALES, 3000) => 2159968881L
- case (WEB_SALES, 10000) => 7199963324L
- case (WEB_SALES, 30000) => 21600036511L
- case (WEB_SALES, 100000) => 71999670164L
- case (t, s) => new Scaling(s).getRowCount(t)
+ def numRows(table: Table, scale: Double): Long = {
+ val nScale = normalize(scale)
+ require(SCALES.contains(nScale), s"Unsupported scale $nScale")
+ (table, nScale) match {
+ case (_, "0") => 0L
+ case (CATALOG_RETURNS, "0.01") => 8923L
+ case (CATALOG_RETURNS, "1") => 144067L
+ case (CATALOG_RETURNS, "10") => 1439749L
+ case (CATALOG_RETURNS, "100") => 14404374L
+ case (CATALOG_RETURNS, "300") => 43193472L
+ case (CATALOG_RETURNS, "1000") => 143996756L
+ case (CATALOG_RETURNS, "3000") => 432018033L
+ case (CATALOG_RETURNS, "10000") => 1440033112L
+ case (CATALOG_RETURNS, "30000") => 4319925093L
+ case (CATALOG_RETURNS, "100000") => 14400175879L
+ case (CATALOG_SALES, "0.01") => 89807L
+ case (CATALOG_SALES, "1") => 1441548L
+ case (CATALOG_SALES, "10") => 14401261L
+ case (CATALOG_SALES, "100") => 143997065L
+ case (CATALOG_SALES, "300") => 431969836L
+ case (CATALOG_SALES, "1000") => 1439980416L
+ case (CATALOG_SALES, "3000") => 4320078880L
+ case (CATALOG_SALES, "10000") => 14399964710L
+ case (CATALOG_SALES, "30000") => 43200404822L
+ case (CATALOG_SALES, "100000") => 143999334399L
+ case (STORE_RETURNS, "0.01") => 11925L
+ case (STORE_RETURNS, "1") => 287514L
+ case (STORE_RETURNS, "10") => 2875432L
+ case (STORE_RETURNS, "100") => 28795080L
+ case (STORE_RETURNS, "300") => 86393244L
+ case (STORE_RETURNS, "1000") => 287999764L
+ case (STORE_RETURNS, "3000") => 863989652L
+ case (STORE_RETURNS, "10000") => 2879970104L
+ case (STORE_RETURNS, "30000") => 8639952111L
+ case (STORE_RETURNS, "100000") => 28800018820L
+ case (STORE_SALES, "0.01") => 120527L
+ case (STORE_SALES, "1") => 2880404L
+ case (STORE_SALES, "10") => 28800991L
+ case (STORE_SALES, "100") => 287997024L
+ case (STORE_SALES, "300") => 864001869L
+ case (STORE_SALES, "1000") => 2879987999L
+ case (STORE_SALES, "3000") => 8639936081L
+ case (STORE_SALES, "10000") => 28799983563L
+ case (STORE_SALES, "30000") => 86399341874L
+ case (STORE_SALES, "100000") => 287997818084L
+ case (WEB_RETURNS, "0.01") => 1152L
+ case (WEB_RETURNS, "1") => 71763L
+ case (WEB_RETURNS, "10") => 719217L
+ case (WEB_RETURNS, "100") => 7197670L
+ case (WEB_RETURNS, "300") => 21599377L
+ case (WEB_RETURNS, "1000") => 71997522L
+ case (WEB_RETURNS, "3000") => 216003761L
+ case (WEB_RETURNS, "10000") => 720020485L
+ case (WEB_RETURNS, "30000") => 2160007345L
+ case (WEB_RETURNS, "100000") => 7199904459L
+ case (WEB_SALES, "0.01") => 11876L
+ case (WEB_SALES, "1") => 719384L
+ case (WEB_SALES, "10") => 7197566L
+ case (WEB_SALES, "100") => 72001237L
+ case (WEB_SALES, "300") => 216009853L
+ case (WEB_SALES, "1000") => 720000376L
+ case (WEB_SALES, "3000") => 2159968881L
+ case (WEB_SALES, "10000") => 7199963324L
+ case (WEB_SALES, "30000") => 21600036511L
+ case (WEB_SALES, "100000") => 71999670164L
+ case (t, s) => new Scaling(s.toDouble).getRowCount(t)
}
}
- def sizeInBytes(table: Table, scale: Int): Long =
+ def sizeInBytes(table: Table, scale: Double): Long =
numRows(table, scale) * TABLE_AVG_ROW_BYTES(table)
private val TABLE_AVG_ROW_BYTES: Map[Table, Long] = Map(
diff --git a/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSTable.scala b/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSTable.scala
index fc0508b6e..701440508 100644
--- a/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSTable.scala
+++ b/extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSTable.scala
@@ -31,7 +31,7 @@ import org.apache.spark.sql.connector.read.ScanBuilder
import org.apache.spark.sql.types._
import org.apache.spark.sql.util.CaseInsensitiveStringMap
-class TPCDSTable(tbl: String, scale: Int, options: CaseInsensitiveStringMap)
+class TPCDSTable(tbl: String, scale: Double, options: CaseInsensitiveStringMap)
extends SparkTable with SupportsRead {
// When true, use CHAR VARCHAR; otherwise use STRING
@@ -47,7 +47,7 @@ class TPCDSTable(tbl: String, scale: Int, options: CaseInsensitiveStringMap)
val tpcdsTable: Table = Table.getTable(tbl)
- override def name: String = s"sf$scale.$tbl"
+ override def name: String = s"${TPCDSSchemaUtils.dbName(scale)}.$tbl"
override def toString: String = s"TPCDSTable($name)"
diff --git a/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSCatalogSuite.scala b/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSCatalogSuite.scala
index e918e14a1..c1bf57d50 100644
--- a/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSCatalogSuite.scala
+++ b/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSCatalogSuite.scala
@@ -44,35 +44,35 @@ class TPCDSCatalogSuite extends KyuubiFunSuite {
test("supports namespaces") {
spark.sql("use tpcds")
- assert(spark.sql(s"SHOW DATABASES").collect().length == 10)
+ assert(spark.sql(s"SHOW DATABASES").collect().length == 11)
assert(spark.sql(s"SHOW NAMESPACES IN tpcds.sf1").collect().length == 0)
}
- test("tpcds.sf1 count") {
- assert(spark.table("tpcds.sf1.call_center").count === 6)
- assert(spark.table("tpcds.sf1.catalog_page").count === 11718)
- assert(spark.table("tpcds.sf1.catalog_returns").count === 144067)
- assert(spark.table("tpcds.sf1.catalog_sales").count === 1441548)
- assert(spark.table("tpcds.sf1.customer").count === 100000)
- assert(spark.table("tpcds.sf1.customer_address").count === 50000)
- assert(spark.table("tpcds.sf1.customer_demographics").count === 1920800)
- assert(spark.table("tpcds.sf1.date_dim").count === 73049)
- assert(spark.table("tpcds.sf1.household_demographics").count === 7200)
- assert(spark.table("tpcds.sf1.income_band").count === 20)
- assert(spark.table("tpcds.sf1.inventory").count === 11745000)
- assert(spark.table("tpcds.sf1.item").count === 18000)
- assert(spark.table("tpcds.sf1.promotion").count === 300)
- assert(spark.table("tpcds.sf1.reason").count === 35)
- assert(spark.table("tpcds.sf1.ship_mode").count === 20)
- assert(spark.table("tpcds.sf1.store").count === 12)
- assert(spark.table("tpcds.sf1.store_returns").count === 287514)
- assert(spark.table("tpcds.sf1.store_sales").count === 2880404)
- assert(spark.table("tpcds.sf1.time_dim").count === 86400)
- assert(spark.table("tpcds.sf1.warehouse").count === 5)
- assert(spark.table("tpcds.sf1.web_page").count === 60)
- assert(spark.table("tpcds.sf1.web_returns").count === 71763)
- assert(spark.table("tpcds.sf1.web_sales").count === 719384)
- assert(spark.table("tpcds.sf1.web_site").count === 30)
+ test("tpcds.tiny count") {
+ assert(spark.table("tpcds.tiny.call_center").count === 2)
+ assert(spark.table("tpcds.tiny.catalog_page").count === 11718)
+ assert(spark.table("tpcds.tiny.catalog_returns").count === 8923)
+ assert(spark.table("tpcds.tiny.catalog_sales").count === 89807)
+ assert(spark.table("tpcds.tiny.customer").count === 1000)
+ assert(spark.table("tpcds.tiny.customer_address").count === 1000)
+ assert(spark.table("tpcds.tiny.customer_demographics").count === 1920800)
+ assert(spark.table("tpcds.tiny.date_dim").count === 73049)
+ assert(spark.table("tpcds.tiny.household_demographics").count === 7200)
+ assert(spark.table("tpcds.tiny.income_band").count === 20)
+ assert(spark.table("tpcds.tiny.inventory").count === 261261)
+ assert(spark.table("tpcds.tiny.item").count === 2000)
+ assert(spark.table("tpcds.tiny.promotion").count === 3)
+ assert(spark.table("tpcds.tiny.reason").count === 1)
+ assert(spark.table("tpcds.tiny.ship_mode").count === 20)
+ assert(spark.table("tpcds.tiny.store").count === 2)
+ assert(spark.table("tpcds.tiny.store_returns").count === 11925)
+ assert(spark.table("tpcds.tiny.store_sales").count === 120527)
+ assert(spark.table("tpcds.tiny.time_dim").count === 86400)
+ assert(spark.table("tpcds.tiny.warehouse").count === 1)
+ assert(spark.table("tpcds.tiny.web_page").count === 2)
+ assert(spark.table("tpcds.tiny.web_returns").count === 1152)
+ assert(spark.table("tpcds.tiny.web_sales").count === 11876)
+ assert(spark.table("tpcds.tiny.web_site").count === 2)
}
test("tpcds.sf1 stats") {
diff --git a/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSSchemaUtilsSuite.scala b/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSSchemaUtilsSuite.scala
new file mode 100644
index 000000000..39737a728
--- /dev/null
+++ b/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSSchemaUtilsSuite.scala
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.spark.connector.tpcds
+
+import org.apache.kyuubi.KyuubiFunSuite
+import org.apache.kyuubi.spark.connector.tpcds.TPCDSSchemaUtils._
+
+class TPCDSSchemaUtilsSuite extends KyuubiFunSuite {
+
+ test("normalize scale") {
+ assert(normalize(1) === "1")
+ assert(normalize(0.010000000000000000001) === "0.01")
+ assert(normalize(1.000000000000000000001) === "1")
+ assert(normalize(0.999999999999999999999) === "1")
+ assert(normalize(9.999999999999999999999) === "10")
+ }
+}
diff --git a/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/spark/kyuubi/TPCDSTableGenerateBenchmark.scala b/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/spark/kyuubi/TPCDSTableGenerateBenchmark.scala
index ab8b046ce..ba8ef64d0 100644
--- a/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/spark/kyuubi/TPCDSTableGenerateBenchmark.scala
+++ b/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/spark/kyuubi/TPCDSTableGenerateBenchmark.scala
@@ -39,7 +39,7 @@ class TPCDSTableGenerateBenchmark extends KyuubiFunSuite with KyuubiBenchmarkBas
private val runBenchmark = sys.env.contains("RUN_BENCHMARK")
private val numRows: Long = 1 * 1000 * 1000
- private val scale = TPCDSStatisticsUtils.SCALES.last
+ private val scale = TPCDSSchemaUtils.SCALES.last.toDouble
test("benchmark") {
assume(runBenchmark)