You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2024/07/09 07:21:43 UTC
(spark) branch master updated: [SPARK-48816][SQL] Shorthand for interval converters in UnivocityParser
This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5adbc03e2809 [SPARK-48816][SQL] Shorthand for interval converters in UnivocityParser
5adbc03e2809 is described below
commit 5adbc03e280920079abd4961d178f7dc19c6b0fb
Author: Kent Yao <ya...@apache.org>
AuthorDate: Tue Jul 9 15:21:29 2024 +0800
[SPARK-48816][SQL] Shorthand for interval converters in UnivocityParser
### What changes were proposed in this pull request?
Directly call `IntervalUtils.castStringToDTInterval/castStringToYMInterval` instead of creating Cast expressions to evaluate.
- Benchmarks indicated a 10% time-saving.
- Bad record recording might not work if the cast handles the exceptions early
### Why are the changes needed?
- pref improvement
- Bugfix for bad record recording
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
passing existing tests and benchmark tests
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #47227 from yaooqinn/SPARK-48816.
Authored-by: Kent Yao <ya...@apache.org>
Signed-off-by: Kent Yao <ya...@apache.org>
---
.../spark/sql/catalyst/csv/UnivocityParser.scala | 8 +-
sql/core/benchmarks/CSVBenchmark-jdk21-results.txt | 89 ++++++++++++----------
sql/core/benchmarks/CSVBenchmark-results.txt | 89 ++++++++++++----------
.../execution/datasources/csv/CSVBenchmark.scala | 33 ++++++++
4 files changed, 134 insertions(+), 85 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
index 61c2f7a5926b..1c9bdfdb493f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
@@ -26,7 +26,7 @@ import com.univocity.parsers.csv.CsvParser
import org.apache.spark.SparkUpgradeException
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, OrderedFilters}
-import org.apache.spark.sql.catalyst.expressions.{Cast, EmptyRow, ExprUtils, GenericInternalRow, Literal}
+import org.apache.spark.sql.catalyst.expressions.{ExprUtils, GenericInternalRow}
import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
import org.apache.spark.sql.errors.{ExecutionErrors, QueryExecutionErrors}
@@ -260,12 +260,14 @@ class UnivocityParser(
case ym: YearMonthIntervalType => (d: String) =>
nullSafeDatum(d, name, nullable, options) { datum =>
- Cast(Literal(datum), ym).eval(EmptyRow)
+ IntervalUtils.castStringToYMInterval(
+ UTF8String.fromString(datum), ym.startField, ym.endField)
}
case dt: DayTimeIntervalType => (d: String) =>
nullSafeDatum(d, name, nullable, options) { datum =>
- Cast(Literal(datum), dt).eval(EmptyRow)
+ IntervalUtils.castStringToDTInterval(
+ UTF8String.fromString(datum), dt.startField, dt.endField)
}
case udt: UserDefinedType[_] =>
diff --git a/sql/core/benchmarks/CSVBenchmark-jdk21-results.txt b/sql/core/benchmarks/CSVBenchmark-jdk21-results.txt
index b37d2fa3060e..50a13bd51ebe 100644
--- a/sql/core/benchmarks/CSVBenchmark-jdk21-results.txt
+++ b/sql/core/benchmarks/CSVBenchmark-jdk21-results.txt
@@ -2,69 +2,76 @@
Benchmark to measure CSV read/write performance
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.3+9-LTS on Linux 6.5.0-1018-azure
+OpenJDK 64-Bit Server VM 21.0.3+9-LTS on Linux 6.5.0-1022-azure
AMD EPYC 7763 64-Core Processor
Parsing quoted values: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-One quoted string 23353 23432 75 0.0 467067.4 1.0X
+One quoted string 23962 24182 316 0.0 479231.3 1.0X
-OpenJDK 64-Bit Server VM 21.0.3+9-LTS on Linux 6.5.0-1018-azure
+OpenJDK 64-Bit Server VM 21.0.3+9-LTS on Linux 6.5.0-1022-azure
AMD EPYC 7763 64-Core Processor
Wide rows with 1000 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Select 1000 columns 56825 57244 679 0.0 56825.1 1.0X
-Select 100 columns 20482 20568 86 0.0 20481.7 2.8X
-Select one column 16968 17000 36 0.1 16967.7 3.3X
-count() 3366 3378 11 0.3 3366.4 16.9X
-Select 100 columns, one bad input field 28347 28379 30 0.0 28346.6 2.0X
-Select 100 columns, corrupt record field 32401 32450 42 0.0 32401.2 1.8X
+Select 1000 columns 56724 57115 570 0.0 56724.1 1.0X
+Select 100 columns 20740 20855 115 0.0 20739.7 2.7X
+Select one column 17304 17377 114 0.1 17304.3 3.3X
+count() 3719 3740 21 0.3 3719.0 15.3X
+Select 100 columns, one bad input field 24943 24999 69 0.0 24943.2 2.3X
+Select 100 columns, corrupt record field 28306 28341 31 0.0 28306.2 2.0X
-OpenJDK 64-Bit Server VM 21.0.3+9-LTS on Linux 6.5.0-1018-azure
+OpenJDK 64-Bit Server VM 21.0.3+9-LTS on Linux 6.5.0-1022-azure
AMD EPYC 7763 64-Core Processor
Count a dataset with 10 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Select 10 columns + count() 11174 11195 18 0.9 1117.4 1.0X
-Select 1 column + count() 7666 7694 24 1.3 766.6 1.5X
-count() 2042 2048 5 4.9 204.2 5.5X
+Select 10 columns + count() 10977 10982 5 0.9 1097.7 1.0X
+Select 1 column + count() 7406 7554 131 1.4 740.6 1.5X
+count() 1550 1558 9 6.5 155.0 7.1X
-OpenJDK 64-Bit Server VM 21.0.3+9-LTS on Linux 6.5.0-1018-azure
+OpenJDK 64-Bit Server VM 21.0.3+9-LTS on Linux 6.5.0-1022-azure
AMD EPYC 7763 64-Core Processor
Write dates and timestamps: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps 854 882 27 11.7 85.4 1.0X
-to_csv(timestamp) 6166 6174 13 1.6 616.6 0.1X
-write timestamps to files 6480 6575 158 1.5 648.0 0.1X
-Create a dataset of dates 948 949 1 10.6 94.8 0.9X
-to_csv(date) 4471 4474 3 2.2 447.1 0.2X
-write dates to files 4599 4616 15 2.2 459.9 0.2X
+Create a dataset of timestamps 845 847 3 11.8 84.5 1.0X
+to_csv(timestamp) 5546 5597 57 1.8 554.6 0.2X
+write timestamps to files 5760 5768 8 1.7 576.0 0.1X
+Create a dataset of dates 1053 1064 9 9.5 105.3 0.8X
+to_csv(date) 4115 4122 9 2.4 411.5 0.2X
+write dates to files 4102 4108 5 2.4 410.2 0.2X
-OpenJDK 64-Bit Server VM 21.0.3+9-LTS on Linux 6.5.0-1018-azure
+OpenJDK 64-Bit Server VM 21.0.3+9-LTS on Linux 6.5.0-1022-azure
AMD EPYC 7763 64-Core Processor
Read dates and timestamps: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files 1200 1213 12 8.3 120.0 1.0X
-read timestamps from files 11576 11601 22 0.9 1157.6 0.1X
-infer timestamps from files 23234 23253 16 0.4 2323.4 0.1X
-read date text from files 1115 1162 44 9.0 111.5 1.1X
-read date from files 10978 11006 43 0.9 1097.8 0.1X
-infer date from files 22588 22604 13 0.4 2258.8 0.1X
-timestamp strings 1224 1236 21 8.2 122.4 1.0X
-parse timestamps from Dataset[String] 13566 13595 41 0.7 1356.6 0.1X
-infer timestamps from Dataset[String] 25057 25094 36 0.4 2505.7 0.0X
-date strings 1618 1626 7 6.2 161.8 0.7X
-parse dates from Dataset[String] 12784 12816 34 0.8 1278.4 0.1X
-from_csv(timestamp) 12008 12088 69 0.8 1200.8 0.1X
-from_csv(date) 11930 11938 12 0.8 1193.0 0.1X
-infer error timestamps from Dataset[String] with default format 14366 14394 35 0.7 1436.6 0.1X
-infer error timestamps from Dataset[String] with user-provided format 14380 14412 52 0.7 1438.0 0.1X
-infer error timestamps from Dataset[String] with legacy format 14439 14453 21 0.7 1443.9 0.1X
+read timestamp text from files 1107 1119 16 9.0 110.7 1.0X
+read timestamps from files 9511 9553 49 1.1 951.1 0.1X
+infer timestamps from files 19084 19114 27 0.5 1908.4 0.1X
+read date text from files 1036 1046 14 9.7 103.6 1.1X
+read date from files 8299 8309 15 1.2 829.9 0.1X
+infer date from files 17290 17294 4 0.6 1729.0 0.1X
+timestamp strings 1188 1197 7 8.4 118.8 0.9X
+parse timestamps from Dataset[String] 11442 11458 14 0.9 1144.2 0.1X
+infer timestamps from Dataset[String] 21076 21116 39 0.5 2107.6 0.1X
+date strings 1651 1659 10 6.1 165.1 0.7X
+parse dates from Dataset[String] 10181 10186 5 1.0 1018.1 0.1X
+from_csv(timestamp) 10023 10062 34 1.0 1002.3 0.1X
+from_csv(date) 9335 9351 15 1.1 933.5 0.1X
+infer error timestamps from Dataset[String] with default format 11187 11205 16 0.9 1118.7 0.1X
+infer error timestamps from Dataset[String] with user-provided format 11201 11216 13 0.9 1120.1 0.1X
+infer error timestamps from Dataset[String] with legacy format 11210 11227 17 0.9 1121.0 0.1X
-OpenJDK 64-Bit Server VM 21.0.3+9-LTS on Linux 6.5.0-1018-azure
+OpenJDK 64-Bit Server VM 21.0.3+9-LTS on Linux 6.5.0-1022-azure
AMD EPYC 7763 64-Core Processor
Filters pushdown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-w/o filters 4302 4383 137 0.0 43020.6 1.0X
-pushdown disabled 4206 4220 13 0.0 42058.8 1.0X
-w/ filters 776 784 10 0.1 7756.3 5.5X
+w/o filters 4365 4377 13 0.0 43653.8 1.0X
+pushdown disabled 4348 4370 22 0.0 43477.7 1.0X
+w/ filters 695 713 29 0.1 6950.2 6.3X
+
+OpenJDK 64-Bit Server VM 21.0.3+9-LTS on Linux 6.5.0-1022-azure
+AMD EPYC 7763 64-Core Processor
+Interval: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Read as Intervals 7089 7096 7 0.4 2362.1 1.0X
+Read Raw Strings 2071 2075 6 1.4 690.1 3.4X
diff --git a/sql/core/benchmarks/CSVBenchmark-results.txt b/sql/core/benchmarks/CSVBenchmark-results.txt
index 522e164f80c8..c1e4f53e1cf4 100644
--- a/sql/core/benchmarks/CSVBenchmark-results.txt
+++ b/sql/core/benchmarks/CSVBenchmark-results.txt
@@ -2,69 +2,76 @@
Benchmark to measure CSV read/write performance
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.11+9-LTS on Linux 6.5.0-1018-azure
+OpenJDK 64-Bit Server VM 17.0.11+9-LTS on Linux 6.5.0-1022-azure
AMD EPYC 7763 64-Core Processor
Parsing quoted values: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-One quoted string 24681 24724 74 0.0 493616.4 1.0X
+One quoted string 25841 26207 446 0.0 516822.6 1.0X
-OpenJDK 64-Bit Server VM 17.0.11+9-LTS on Linux 6.5.0-1018-azure
+OpenJDK 64-Bit Server VM 17.0.11+9-LTS on Linux 6.5.0-1022-azure
AMD EPYC 7763 64-Core Processor
Wide rows with 1000 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Select 1000 columns 55362 55719 576 0.0 55361.6 1.0X
-Select 100 columns 22947 22975 36 0.0 22946.7 2.4X
-Select one column 19695 19714 18 0.1 19694.7 2.8X
-count() 3474 3514 54 0.3 3473.8 15.9X
-Select 100 columns, one bad input field 32366 32417 47 0.0 32365.6 1.7X
-Select 100 columns, corrupt record field 35921 35986 77 0.0 35921.3 1.5X
+Select 1000 columns 57462 57944 502 0.0 57462.5 1.0X
+Select 100 columns 23373 23449 71 0.0 23372.8 2.5X
+Select one column 20009 20041 40 0.0 20008.9 2.9X
+count() 4326 4402 66 0.2 4325.9 13.3X
+Select 100 columns, one bad input field 28362 28472 104 0.0 28361.6 2.0X
+Select 100 columns, corrupt record field 31926 32002 113 0.0 31925.7 1.8X
-OpenJDK 64-Bit Server VM 17.0.11+9-LTS on Linux 6.5.0-1018-azure
+OpenJDK 64-Bit Server VM 17.0.11+9-LTS on Linux 6.5.0-1022-azure
AMD EPYC 7763 64-Core Processor
Count a dataset with 10 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Select 10 columns + count() 9523 9537 15 1.1 952.3 1.0X
-Select 1 column + count() 6868 6883 13 1.5 686.8 1.4X
-count() 1820 1836 20 5.5 182.0 5.2X
+Select 10 columns + count() 9591 9615 23 1.0 959.1 1.0X
+Select 1 column + count() 6827 6845 21 1.5 682.7 1.4X
+count() 1754 1759 5 5.7 175.4 5.5X
-OpenJDK 64-Bit Server VM 17.0.11+9-LTS on Linux 6.5.0-1018-azure
+OpenJDK 64-Bit Server VM 17.0.11+9-LTS on Linux 6.5.0-1022-azure
AMD EPYC 7763 64-Core Processor
Write dates and timestamps: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps 899 912 12 11.1 89.9 1.0X
-to_csv(timestamp) 7355 7371 14 1.4 735.5 0.1X
-write timestamps to files 7751 7761 12 1.3 775.1 0.1X
-Create a dataset of dates 1171 1174 6 8.5 117.1 0.8X
-to_csv(date) 5040 5044 4 2.0 504.0 0.2X
-write dates to files 5277 5292 24 1.9 527.7 0.2X
+Create a dataset of timestamps 932 938 9 10.7 93.2 1.0X
+to_csv(timestamp) 7214 7275 83 1.4 721.4 0.1X
+write timestamps to files 7593 7604 10 1.3 759.3 0.1X
+Create a dataset of dates 1036 1042 7 9.6 103.6 0.9X
+to_csv(date) 5121 5139 15 2.0 512.1 0.2X
+write dates to files 5203 5215 16 1.9 520.3 0.2X
-OpenJDK 64-Bit Server VM 17.0.11+9-LTS on Linux 6.5.0-1018-azure
+OpenJDK 64-Bit Server VM 17.0.11+9-LTS on Linux 6.5.0-1022-azure
AMD EPYC 7763 64-Core Processor
Read dates and timestamps: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files 1245 1251 7 8.0 124.5 1.0X
-read timestamps from files 10059 10074 23 1.0 1005.9 0.1X
-infer timestamps from files 20189 20223 36 0.5 2018.9 0.1X
-read date text from files 1151 1167 24 8.7 115.1 1.1X
-read date from files 10547 10568 25 0.9 1054.7 0.1X
-infer date from files 21527 21540 11 0.5 2152.7 0.1X
-timestamp strings 1355 1364 15 7.4 135.5 0.9X
-parse timestamps from Dataset[String] 11522 11553 28 0.9 1152.2 0.1X
-infer timestamps from Dataset[String] 21195 21203 10 0.5 2119.5 0.1X
-date strings 1785 1788 5 5.6 178.5 0.7X
-parse dates from Dataset[String] 12245 12288 44 0.8 1224.5 0.1X
-from_csv(timestamp) 10102 10144 51 1.0 1010.2 0.1X
-from_csv(date) 11329 11353 29 0.9 1132.9 0.1X
-infer error timestamps from Dataset[String] with default format 12067 12091 36 0.8 1206.7 0.1X
-infer error timestamps from Dataset[String] with user-provided format 12077 12093 24 0.8 1207.7 0.1X
-infer error timestamps from Dataset[String] with legacy format 12047 12076 26 0.8 1204.7 0.1X
+read timestamp text from files 1258 1260 3 7.9 125.8 1.0X
+read timestamps from files 10113 10133 17 1.0 1011.3 0.1X
+infer timestamps from files 19797 19926 134 0.5 1979.7 0.1X
+read date text from files 1131 1133 3 8.8 113.1 1.1X
+read date from files 10275 10283 8 1.0 1027.5 0.1X
+infer date from files 20823 20856 29 0.5 2082.3 0.1X
+timestamp strings 1330 1335 7 7.5 133.0 0.9X
+parse timestamps from Dataset[String] 11297 11345 43 0.9 1129.7 0.1X
+infer timestamps from Dataset[String] 20968 20999 41 0.5 2096.8 0.1X
+date strings 1786 1793 7 5.6 178.6 0.7X
+parse dates from Dataset[String] 11565 11595 34 0.9 1156.5 0.1X
+from_csv(timestamp) 9920 9948 42 1.0 992.0 0.1X
+from_csv(date) 10506 10512 5 1.0 1050.6 0.1X
+infer error timestamps from Dataset[String] with default format 12344 12370 23 0.8 1234.4 0.1X
+infer error timestamps from Dataset[String] with user-provided format 12351 12367 17 0.8 1235.1 0.1X
+infer error timestamps from Dataset[String] with legacy format 12345 12366 18 0.8 1234.5 0.1X
-OpenJDK 64-Bit Server VM 17.0.11+9-LTS on Linux 6.5.0-1018-azure
+OpenJDK 64-Bit Server VM 17.0.11+9-LTS on Linux 6.5.0-1022-azure
AMD EPYC 7763 64-Core Processor
Filters pushdown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-w/o filters 4119 4139 17 0.0 41191.2 1.0X
-pushdown disabled 4092 4110 16 0.0 40922.3 1.0X
-w/ filters 691 702 13 0.1 6911.5 6.0X
+w/o filters 4103 4108 5 0.0 41030.6 1.0X
+pushdown disabled 4129 4132 4 0.0 41291.7 1.0X
+w/ filters 769 775 6 0.1 7689.8 5.3X
+
+OpenJDK 64-Bit Server VM 17.0.11+9-LTS on Linux 6.5.0-1022-azure
+AMD EPYC 7763 64-Core Processor
+Interval: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Read as Intervals 6922 6964 44 0.4 2306.6 1.0X
+Read Raw Strings 2360 2391 27 1.3 786.5 2.9X
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVBenchmark.scala
index e9cf35d9fab9..524c22206215 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVBenchmark.scala
@@ -371,6 +371,38 @@ object CSVBenchmark extends SqlBasedBenchmark {
}
}
+ private def intervalBenchmark(rowsNum: Int, numIters: Int): Unit = {
+ val benchmark = new Benchmark(s"Interval", rowsNum, output = output)
+ withTempPath { path =>
+ spark
+ .range(rowsNum)
+ .map { i =>
+ (s"${i % 1000}-${"%02d".format(i % 12)}",
+ s"${i % 1000} " +
+ s"${"%02d".format(i % 24)}:" +
+ s"${"%02d".format(i % 60)}:" +
+ s"${"%02d".format(i % 60)}.${i % 1000000}")
+ }
+ .toDF("ym", "ds")
+ .write
+ .option("header", true)
+ .mode("overwrite")
+ .csv(path.getAbsolutePath)
+ benchmark.addCase("Read as Intervals", numIters) { _ =>
+ spark.read.option("header", true)
+ .schema("ym INTERVAL YEAR TO MONTH, ds INTERVAL DAY TO SECOND")
+ .csv(path.getAbsolutePath)
+ .noop()
+ }
+ benchmark.addCase("Read Raw Strings", numIters) { _ =>
+ spark.read.option("header", true)
+ .csv(path.getAbsolutePath)
+ .noop()
+ }
+ benchmark.run()
+ }
+ }
+
override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
runBenchmark("Benchmark to measure CSV read/write performance") {
val numIters = 3
@@ -379,6 +411,7 @@ object CSVBenchmark extends SqlBasedBenchmark {
countBenchmark(rowsNum = 10 * 1000 * 1000, numIters)
datetimeBenchmark(rowsNum = 10 * 1000 * 1000, numIters)
filtersPushdownBenchmark(rowsNum = 100 * 1000, numIters)
+ intervalBenchmark(rowsNum = 300 * 1000, numIters)
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org