You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2018/10/02 17:04:56 UTC
spark git commit: [SPARK-25581][SQL] Rename method `benchmark` as
`runBenchmarkSuite` in `BenchmarkBase`
Repository: spark
Updated Branches:
refs/heads/master 9bf397c0e -> 7b4e94f16
[SPARK-25581][SQL] Rename method `benchmark` as `runBenchmarkSuite` in `BenchmarkBase`
## What changes were proposed in this pull request?
Rename method `benchmark` in `BenchmarkBase` as `runBenchmarkSuite `. Also add comments.
Currently the method name `benchmark` is a bit confusing. Also the name is the same as instances of `Benchmark`:
https://github.com/apache/spark/blob/f246813afba16fee4d703f09e6302011b11806f3/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcReadBenchmark.scala#L330-L339
## How was this patch tested?
Unit test.
Closes #22599 from gengliangwang/renameBenchmarkSuite.
Authored-by: Gengliang Wang <ge...@databricks.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7b4e94f1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7b4e94f1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7b4e94f1
Branch: refs/heads/master
Commit: 7b4e94f16096cd35835450d63620583496e4f978
Parents: 9bf397c
Author: Gengliang Wang <ge...@databricks.com>
Authored: Tue Oct 2 10:04:47 2018 -0700
Committer: Dongjoon Hyun <do...@apache.org>
Committed: Tue Oct 2 10:04:47 2018 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/benchmark/BenchmarkBase.scala | 9 +++++++--
.../spark/mllib/linalg/UDTSerializationBenchmark.scala | 2 +-
.../org/apache/spark/sql/UnsafeProjectionBenchmark.scala | 2 +-
.../spark/sql/execution/benchmark/AggregateBenchmark.scala | 2 +-
.../sql/execution/benchmark/FilterPushdownBenchmark.scala | 2 +-
.../sql/execution/benchmark/PrimitiveArrayBenchmark.scala | 2 +-
.../spark/sql/execution/benchmark/SortBenchmark.scala | 2 +-
.../columnar/compression/CompressionSchemeBenchmark.scala | 2 +-
.../sql/execution/vectorized/ColumnarBatchBenchmark.scala | 2 +-
.../org/apache/spark/sql/hive/orc/OrcReadBenchmark.scala | 2 +-
10 files changed, 16 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/7b4e94f1/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala b/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala
index 9a37e02..89e927e 100644
--- a/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala
+++ b/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala
@@ -25,7 +25,12 @@ import java.io.{File, FileOutputStream, OutputStream}
abstract class BenchmarkBase {
var output: Option[OutputStream] = None
- def benchmark(): Unit
+ /**
+ * Main process of the whole benchmark.
+ * Implementations of this method are supposed to use the wrapper method `runBenchmark`
+ * for each benchmark scenario.
+ */
+ def runBenchmarkSuite(): Unit
final def runBenchmark(benchmarkName: String)(func: => Any): Unit = {
val separator = "=" * 96
@@ -46,7 +51,7 @@ abstract class BenchmarkBase {
output = Some(new FileOutputStream(file))
}
- benchmark()
+ runBenchmarkSuite()
output.foreach { o =>
if (o != null) {
http://git-wip-us.apache.org/repos/asf/spark/blob/7b4e94f1/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
index 1a2216e..6c1d580 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
@@ -32,7 +32,7 @@ import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
*/
object UDTSerializationBenchmark extends BenchmarkBase {
- override def benchmark(): Unit = {
+ override def runBenchmarkSuite(): Unit = {
runBenchmark("VectorUDT de/serialization") {
val iters = 1e2.toInt
http://git-wip-us.apache.org/repos/asf/spark/blob/7b4e94f1/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala
index cbe723f..e7a9948 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala
@@ -41,7 +41,7 @@ object UnsafeProjectionBenchmark extends BenchmarkBase {
(1 to numRows).map(_ => encoder.toRow(generator().asInstanceOf[Row]).copy()).toArray
}
- override def benchmark(): Unit = {
+ override def runBenchmarkSuite(): Unit = {
runBenchmark("unsafe projection") {
val iters = 1024 * 16
val numRows = 1024 * 16
http://git-wip-us.apache.org/repos/asf/spark/blob/7b4e94f1/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/AggregateBenchmark.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/AggregateBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/AggregateBenchmark.scala
index 296ae10..86e0df2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/AggregateBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/AggregateBenchmark.scala
@@ -44,7 +44,7 @@ import org.apache.spark.unsafe.map.BytesToBytesMap
*/
object AggregateBenchmark extends SqlBasedBenchmark {
- override def benchmark(): Unit = {
+ override def runBenchmarkSuite(): Unit = {
runBenchmark("aggregate without grouping") {
val N = 500L << 22
codegenBenchmark("agg w/o group", N) {
http://git-wip-us.apache.org/repos/asf/spark/blob/7b4e94f1/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala
index 7cdf653..cf05ca3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala
@@ -198,7 +198,7 @@ object FilterPushdownBenchmark extends BenchmarkBase with SQLHelper {
}
}
- override def benchmark(): Unit = {
+ override def runBenchmarkSuite(): Unit = {
runBenchmark("Pushdown for many distinct value case") {
withTempPath { dir =>
withTempTable("orcTable", "parquetTable") {
http://git-wip-us.apache.org/repos/asf/spark/blob/7b4e94f1/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala
index 8b27518..83edf73 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala
@@ -36,7 +36,7 @@ object PrimitiveArrayBenchmark extends BenchmarkBase {
.config("spark.sql.autoBroadcastJoinThreshold", 1)
.getOrCreate()
- override def benchmark(): Unit = {
+ override def runBenchmarkSuite(): Unit = {
runBenchmark("Write primitive arrays in dataset") {
writeDatasetArray(4)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/7b4e94f1/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SortBenchmark.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SortBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SortBenchmark.scala
index 958a064..9a54e23 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SortBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SortBenchmark.scala
@@ -119,7 +119,7 @@ object SortBenchmark extends BenchmarkBase {
benchmark.run()
}
- override def benchmark(): Unit = {
+ override def runBenchmarkSuite(): Unit = {
runBenchmark("radix sort") {
sortBenchmark()
}
http://git-wip-us.apache.org/repos/asf/spark/blob/7b4e94f1/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/CompressionSchemeBenchmark.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/CompressionSchemeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/CompressionSchemeBenchmark.scala
index ff0e4ac..0f90797 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/CompressionSchemeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/CompressionSchemeBenchmark.scala
@@ -233,7 +233,7 @@ object CompressionSchemeBenchmark extends BenchmarkBase with AllCompressionSchem
runDecodeBenchmark("STRING Decode", iters, count, STRING, testData)
}
- override def benchmark(): Unit = {
+ override def runBenchmarkSuite(): Unit = {
runBenchmark("Compression Scheme Benchmark") {
bitEncodingBenchmark(1024)
shortEncodingBenchmark(1024)
http://git-wip-us.apache.org/repos/asf/spark/blob/7b4e94f1/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchBenchmark.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchBenchmark.scala
index df6ab14..f311465 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchBenchmark.scala
@@ -443,7 +443,7 @@ object ColumnarBatchBenchmark extends BenchmarkBase {
benchmark.run
}
- override def benchmark(): Unit = {
+ override def runBenchmarkSuite(): Unit = {
runBenchmark("Int Read/Write") {
intAccess(1024 * 40)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/7b4e94f1/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcReadBenchmark.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcReadBenchmark.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcReadBenchmark.scala
index 0bb5e8c..870ad48 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcReadBenchmark.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcReadBenchmark.scala
@@ -336,7 +336,7 @@ object OrcReadBenchmark extends BenchmarkBase with SQLHelper {
}
}
- override def benchmark(): Unit = {
+ override def runBenchmarkSuite(): Unit = {
runBenchmark("SQL Single Numeric Column Scan") {
Seq(ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType).foreach { dataType =>
numericScanBenchmark(1024 * 1024 * 15, dataType)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org