You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2018/09/24 03:46:57 UTC
spark git commit: [SPARK-25478][SQL][TEST] Refactor
CompressionSchemeBenchmark to use main method
Repository: spark
Updated Branches:
refs/heads/master d522a563a -> c79072aaf
[SPARK-25478][SQL][TEST] Refactor CompressionSchemeBenchmark to use main method
## What changes were proposed in this pull request?
Refactor `CompressionSchemeBenchmark` to use main method.
Generate benchmark result:
```sh
SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain org.apache.spark.sql.execution.columnar.compression.CompressionSchemeBenchmark"
```
## How was this patch tested?
manual tests
Closes #22486 from wangyum/SPARK-25478.
Lead-authored-by: Yuming Wang <yu...@ebay.com>
Co-authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c79072aa
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c79072aa
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c79072aa
Branch: refs/heads/master
Commit: c79072aafa2f406c342e393e0c61bb5cb3e89a7f
Parents: d522a56
Author: Yuming Wang <yu...@ebay.com>
Authored: Sun Sep 23 20:46:40 2018 -0700
Committer: Dongjoon Hyun <do...@apache.org>
Committed: Sun Sep 23 20:46:40 2018 -0700
----------------------------------------------------------------------
.../CompressionSchemeBenchmark-results.txt | 137 ++++++++++++++++++
.../CompressionSchemeBenchmark.scala | 138 +++----------------
2 files changed, 156 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c79072aa/sql/core/benchmarks/CompressionSchemeBenchmark-results.txt
----------------------------------------------------------------------
diff --git a/sql/core/benchmarks/CompressionSchemeBenchmark-results.txt b/sql/core/benchmarks/CompressionSchemeBenchmark-results.txt
new file mode 100644
index 0000000..caa9378
--- /dev/null
+++ b/sql/core/benchmarks/CompressionSchemeBenchmark-results.txt
@@ -0,0 +1,137 @@
+================================================================================================
+Compression Scheme Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+BOOLEAN Encode: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough(1.000) 4 / 4 17998.9 0.1 1.0X
+RunLengthEncoding(2.501) 680 / 680 98.7 10.1 0.0X
+BooleanBitSet(0.125) 365 / 365 183.9 5.4 0.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+BOOLEAN Decode: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough 144 / 144 466.5 2.1 1.0X
+RunLengthEncoding 679 / 679 98.9 10.1 0.2X
+BooleanBitSet 1425 / 1431 47.1 21.2 0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+SHORT Encode (Lower Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough(1.000) 7 / 7 10115.0 0.1 1.0X
+RunLengthEncoding(1.494) 1671 / 1672 40.2 24.9 0.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+SHORT Decode (Lower Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough 1128 / 1128 59.5 16.8 1.0X
+RunLengthEncoding 1630 / 1633 41.2 24.3 0.7X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+SHORT Encode (Higher Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough(1.000) 7 / 7 10164.2 0.1 1.0X
+RunLengthEncoding(1.989) 1562 / 1563 43.0 23.3 0.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+SHORT Decode (Higher Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough 1127 / 1127 59.6 16.8 1.0X
+RunLengthEncoding 1629 / 1631 41.2 24.3 0.7X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+INT Encode (Lower Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough(1.000) 22 / 23 2983.2 0.3 1.0X
+RunLengthEncoding(1.003) 2426 / 2427 27.7 36.1 0.0X
+DictionaryEncoding(0.500) 958 / 958 70.1 14.3 0.0X
+IntDelta(0.250) 286 / 286 235.0 4.3 0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+INT Decode (Lower Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough 1268 / 1269 52.9 18.9 1.0X
+RunLengthEncoding 1906 / 1911 35.2 28.4 0.7X
+DictionaryEncoding 981 / 982 68.4 14.6 1.3X
+IntDelta 812 / 817 82.6 12.1 1.6X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+INT Encode (Higher Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough(1.000) 23 / 23 2926.9 0.3 1.0X
+RunLengthEncoding(1.326) 2614 / 2614 25.7 38.9 0.0X
+DictionaryEncoding(0.501) 1024 / 1024 65.5 15.3 0.0X
+IntDelta(0.250) 286 / 286 234.7 4.3 0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+INT Decode (Higher Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough 1433 / 1433 46.8 21.4 1.0X
+RunLengthEncoding 1923 / 1926 34.9 28.6 0.7X
+DictionaryEncoding 1285 / 1285 52.2 19.2 1.1X
+IntDelta 1129 / 1137 59.4 16.8 1.3X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+LONG Encode (Lower Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough(1.000) 45 / 45 1495.6 0.7 1.0X
+RunLengthEncoding(0.738) 2662 / 2663 25.2 39.7 0.0X
+DictionaryEncoding(0.250) 1269 / 1269 52.9 18.9 0.0X
+LongDelta(0.125) 450 / 450 149.1 6.7 0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+LONG Decode (Lower Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough 1483 / 1483 45.3 22.1 1.0X
+RunLengthEncoding 1875 / 1875 35.8 27.9 0.8X
+DictionaryEncoding 1213 / 1214 55.3 18.1 1.2X
+LongDelta 816 / 817 82.2 12.2 1.8X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+LONG Encode (Higher Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough(1.000) 45 / 45 1489.3 0.7 1.0X
+RunLengthEncoding(1.003) 2906 / 2906 23.1 43.3 0.0X
+DictionaryEncoding(0.251) 1610 / 1610 41.7 24.0 0.0X
+LongDelta(0.125) 451 / 451 148.7 6.7 0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+LONG Decode (Higher Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough 1485 / 1485 45.2 22.1 1.0X
+RunLengthEncoding 1889 / 1890 35.5 28.2 0.8X
+DictionaryEncoding 1215 / 1216 55.2 18.1 1.2X
+LongDelta 1107 / 1110 60.6 16.5 1.3X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+STRING Encode: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough(1.000) 67 / 68 994.5 1.0 1.0X
+RunLengthEncoding(0.894) 5877 / 5882 11.4 87.6 0.0X
+DictionaryEncoding(0.167) 3597 / 3602 18.7 53.6 0.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+STRING Decode: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------
+PassThrough 3243 / 3244 20.7 48.3 1.0X
+RunLengthEncoding 3598 / 3601 18.7 53.6 0.9X
+DictionaryEncoding 3182 / 3182 21.1 47.4 1.0X
+
+
http://git-wip-us.apache.org/repos/asf/spark/blob/c79072aa/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/CompressionSchemeBenchmark.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/CompressionSchemeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/CompressionSchemeBenchmark.scala
index 9c26d67..ff0e4ac 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/CompressionSchemeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/CompressionSchemeBenchmark.scala
@@ -23,7 +23,7 @@ import java.nio.charset.StandardCharsets
import org.apache.commons.lang3.RandomStringUtils
import org.apache.commons.math3.distribution.LogNormalDistribution
-import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
import org.apache.spark.sql.execution.columnar.{BOOLEAN, INT, LONG, NativeColumnType, SHORT, STRING}
import org.apache.spark.sql.types.AtomicType
@@ -31,8 +31,15 @@ import org.apache.spark.util.Utils._
/**
* Benchmark to decoders using various compression schemes.
+ * To run this benchmark:
+ * {{{
+ * 1. without sbt: bin/spark-submit --class <this class> <spark sql test jar>
+ * 2. build/sbt "sql/test:runMain <this class>"
+ * 3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
+ * Results will be written to "benchmarks/CompressionSchemeBenchmark-results.txt".
+ * }}}
*/
-object CompressionSchemeBenchmark extends AllCompressionSchemes {
+object CompressionSchemeBenchmark extends BenchmarkBase with AllCompressionSchemes {
private[this] def allocateLocal(size: Int): ByteBuffer = {
ByteBuffer.allocate(size).order(ByteOrder.nativeOrder)
@@ -77,7 +84,7 @@ object CompressionSchemeBenchmark extends AllCompressionSchemes {
count: Int,
tpe: NativeColumnType[T],
input: ByteBuffer): Unit = {
- val benchmark = new Benchmark(name, iters * count.toLong)
+ val benchmark = new Benchmark(name, iters * count.toLong, output = output)
schemes.filter(_.supports(tpe)).foreach { scheme =>
val (compressFunc, compressionRatio, buf) = prepareEncodeInternal(count, tpe, scheme, input)
@@ -101,7 +108,7 @@ object CompressionSchemeBenchmark extends AllCompressionSchemes {
count: Int,
tpe: NativeColumnType[T],
input: ByteBuffer): Unit = {
- val benchmark = new Benchmark(name, iters * count.toLong)
+ val benchmark = new Benchmark(name, iters * count.toLong, output = output)
schemes.filter(_.supports(tpe)).foreach { scheme =>
val (compressFunc, _, buf) = prepareEncodeInternal(count, tpe, scheme, input)
@@ -138,21 +145,7 @@ object CompressionSchemeBenchmark extends AllCompressionSchemes {
testData.put(i * BOOLEAN.defaultSize, g())
}
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // BOOLEAN Encode: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough(1.000) 3 / 4 19300.2 0.1 1.0X
- // RunLengthEncoding(2.491) 923 / 939 72.7 13.8 0.0X
- // BooleanBitSet(0.125) 359 / 363 187.1 5.3 0.0X
runEncodeBenchmark("BOOLEAN Encode", iters, count, BOOLEAN, testData)
-
-
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // BOOLEAN Decode: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough 129 / 136 519.8 1.9 1.0X
- // RunLengthEncoding 613 / 623 109.4 9.1 0.2X
- // BooleanBitSet 1196 / 1222 56.1 17.8 0.1X
runDecodeBenchmark("BOOLEAN Decode", iters, count, BOOLEAN, testData)
}
@@ -165,18 +158,7 @@ object CompressionSchemeBenchmark extends AllCompressionSchemes {
testData.putShort(i * SHORT.defaultSize, g1().toShort)
}
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // SHORT Encode (Lower Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough(1.000) 6 / 7 10971.4 0.1 1.0X
- // RunLengthEncoding(1.510) 1526 / 1542 44.0 22.7 0.0X
runEncodeBenchmark("SHORT Encode (Lower Skew)", iters, count, SHORT, testData)
-
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // SHORT Decode (Lower Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough 811 / 837 82.8 12.1 1.0X
- // RunLengthEncoding 1219 / 1266 55.1 18.2 0.7X
runDecodeBenchmark("SHORT Decode (Lower Skew)", iters, count, SHORT, testData)
val g2 = genHigherSkewData()
@@ -184,18 +166,7 @@ object CompressionSchemeBenchmark extends AllCompressionSchemes {
testData.putShort(i * SHORT.defaultSize, g2().toShort)
}
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // SHORT Encode (Higher Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough(1.000) 7 / 7 10112.4 0.1 1.0X
- // RunLengthEncoding(2.009) 1623 / 1661 41.4 24.2 0.0X
runEncodeBenchmark("SHORT Encode (Higher Skew)", iters, count, SHORT, testData)
-
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // SHORT Decode (Higher Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough 818 / 827 82.0 12.2 1.0X
- // RunLengthEncoding 1202 / 1237 55.8 17.9 0.7X
runDecodeBenchmark("SHORT Decode (Higher Skew)", iters, count, SHORT, testData)
}
@@ -208,22 +179,7 @@ object CompressionSchemeBenchmark extends AllCompressionSchemes {
testData.putInt(i * INT.defaultSize, g1().toInt)
}
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // INT Encode (Lower Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough(1.000) 18 / 19 3716.4 0.3 1.0X
- // RunLengthEncoding(1.001) 1992 / 2056 33.7 29.7 0.0X
- // DictionaryEncoding(0.500) 723 / 739 92.8 10.8 0.0X
- // IntDelta(0.250) 368 / 377 182.2 5.5 0.0X
runEncodeBenchmark("INT Encode (Lower Skew)", iters, count, INT, testData)
-
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // INT Decode (Lower Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough 821 / 845 81.8 12.2 1.0X
- // RunLengthEncoding 1246 / 1256 53.9 18.6 0.7X
- // DictionaryEncoding 757 / 766 88.6 11.3 1.1X
- // IntDelta 680 / 689 98.7 10.1 1.2X
runDecodeBenchmark("INT Decode (Lower Skew)", iters, count, INT, testData)
val g2 = genHigherSkewData()
@@ -231,22 +187,7 @@ object CompressionSchemeBenchmark extends AllCompressionSchemes {
testData.putInt(i * INT.defaultSize, g2().toInt)
}
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // INT Encode (Higher Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough(1.000) 17 / 19 3888.4 0.3 1.0X
- // RunLengthEncoding(1.339) 2127 / 2148 31.5 31.7 0.0X
- // DictionaryEncoding(0.501) 960 / 972 69.9 14.3 0.0X
- // IntDelta(0.250) 362 / 366 185.5 5.4 0.0X
runEncodeBenchmark("INT Encode (Higher Skew)", iters, count, INT, testData)
-
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // INT Decode (Higher Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough 838 / 884 80.1 12.5 1.0X
- // RunLengthEncoding 1287 / 1311 52.1 19.2 0.7X
- // DictionaryEncoding 844 / 859 79.5 12.6 1.0X
- // IntDelta 764 / 784 87.8 11.4 1.1X
runDecodeBenchmark("INT Decode (Higher Skew)", iters, count, INT, testData)
}
@@ -259,22 +200,7 @@ object CompressionSchemeBenchmark extends AllCompressionSchemes {
testData.putLong(i * LONG.defaultSize, g1().toLong)
}
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // LONG Encode (Lower Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough(1.000) 37 / 38 1804.8 0.6 1.0X
- // RunLengthEncoding(0.748) 2065 / 2094 32.5 30.8 0.0X
- // DictionaryEncoding(0.250) 950 / 962 70.6 14.2 0.0X
- // LongDelta(0.125) 475 / 482 141.2 7.1 0.1X
runEncodeBenchmark("LONG Encode (Lower Skew)", iters, count, LONG, testData)
-
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // LONG Decode (Lower Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough 888 / 894 75.5 13.2 1.0X
- // RunLengthEncoding 1301 / 1311 51.6 19.4 0.7X
- // DictionaryEncoding 887 / 904 75.7 13.2 1.0X
- // LongDelta 693 / 735 96.8 10.3 1.3X
runDecodeBenchmark("LONG Decode (Lower Skew)", iters, count, LONG, testData)
val g2 = genHigherSkewData()
@@ -282,22 +208,7 @@ object CompressionSchemeBenchmark extends AllCompressionSchemes {
testData.putLong(i * LONG.defaultSize, g2().toLong)
}
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // LONG Encode (Higher Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough(1.000) 34 / 35 1963.9 0.5 1.0X
- // RunLengthEncoding(0.999) 2260 / 3021 29.7 33.7 0.0X
- // DictionaryEncoding(0.251) 1270 / 1438 52.8 18.9 0.0X
- // LongDelta(0.125) 496 / 509 135.3 7.4 0.1X
runEncodeBenchmark("LONG Encode (Higher Skew)", iters, count, LONG, testData)
-
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // LONG Decode (Higher Skew): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough 965 / 1494 69.5 14.4 1.0X
- // RunLengthEncoding 1350 / 1378 49.7 20.1 0.7X
- // DictionaryEncoding 892 / 924 75.2 13.3 1.1X
- // LongDelta 817 / 847 82.2 12.2 1.2X
runDecodeBenchmark("LONG Decode (Higher Skew)", iters, count, LONG, testData)
}
@@ -318,28 +229,17 @@ object CompressionSchemeBenchmark extends AllCompressionSchemes {
}
testData.rewind()
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // STRING Encode: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough(1.000) 56 / 57 1197.9 0.8 1.0X
- // RunLengthEncoding(0.893) 4892 / 4937 13.7 72.9 0.0X
- // DictionaryEncoding(0.167) 2968 / 2992 22.6 44.2 0.0X
runEncodeBenchmark("STRING Encode", iters, count, STRING, testData)
-
- // Intel(R) Core(TM) i7-4578U CPU @ 3.00GHz
- // STRING Decode: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
- // -------------------------------------------------------------------------------------------
- // PassThrough 2422 / 2449 27.7 36.1 1.0X
- // RunLengthEncoding 2885 / 3018 23.3 43.0 0.8X
- // DictionaryEncoding 2716 / 2752 24.7 40.5 0.9X
runDecodeBenchmark("STRING Decode", iters, count, STRING, testData)
}
- def main(args: Array[String]): Unit = {
- bitEncodingBenchmark(1024)
- shortEncodingBenchmark(1024)
- intEncodingBenchmark(1024)
- longEncodingBenchmark(1024)
- stringEncodingBenchmark(1024)
+ override def benchmark(): Unit = {
+ runBenchmark("Compression Scheme Benchmark") {
+ bitEncodingBenchmark(1024)
+ shortEncodingBenchmark(1024)
+ intEncodingBenchmark(1024)
+ longEncodingBenchmark(1024)
+ stringEncodingBenchmark(1024)
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org