You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sa...@apache.org on 2018/02/05 18:19:50 UTC
spark git commit: [SPARK-23310][CORE] Turn off read ahead input
stream for unshafe shuffle reader
Repository: spark
Updated Branches:
refs/heads/master a6bf3db20 -> 03b7e120d
[SPARK-23310][CORE] Turn off read ahead input stream for unshafe shuffle reader
To fix regression for TPC-DS queries
Author: Sital Kedia <sk...@fb.com>
Closes #20492 from sitalkedia/turn_off_async_inputstream.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/03b7e120
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/03b7e120
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/03b7e120
Branch: refs/heads/master
Commit: 03b7e120dd7ff7848c936c7a23644da5bd7219ab
Parents: a6bf3db
Author: Sital Kedia <sk...@fb.com>
Authored: Mon Feb 5 10:19:18 2018 -0800
Committer: Sameer Agarwal <sa...@apache.org>
Committed: Mon Feb 5 10:19:18 2018 -0800
----------------------------------------------------------------------
.../util/collection/unsafe/sort/UnsafeSorterSpillReader.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/03b7e120/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
index e2f48e5..71e7c7a 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
@@ -76,8 +76,10 @@ public final class UnsafeSorterSpillReader extends UnsafeSorterIterator implemen
SparkEnv.get() == null ? 0.5 :
SparkEnv.get().conf().getDouble("spark.unsafe.sorter.spill.read.ahead.fraction", 0.5);
+ // SPARK-23310: Disable read-ahead input stream, because it is causing lock contention and perf regression for
+ // TPC-DS queries.
final boolean readAheadEnabled = SparkEnv.get() != null &&
- SparkEnv.get().conf().getBoolean("spark.unsafe.sorter.spill.read.ahead.enabled", true);
+ SparkEnv.get().conf().getBoolean("spark.unsafe.sorter.spill.read.ahead.enabled", false);
final InputStream bs =
new NioBufferedFileInputStream(file, (int) bufferSizeBytes);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org