You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by mr...@apache.org on 2022/08/18 14:59:48 UTC
[spark] branch master updated (9381fe6891d -> 892a45f22e5)
This is an automated email from the ASF dual-hosted git repository.
mridulm80 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
from 9381fe6891d [SPARK-39975][BUILD] Upgrade rocksdbjni to 7.4.5
add 892a45f22e5 [SPARK-38909][CORE][YARN] Encapsulate `LevelDB` used to store remote/external shuffle state as `DB`
No new revisions were added by this update.
Summary of changes:
.../org/apache/spark/network/shuffledb/DB.java | 37 ++++++---
.../apache/spark/network/shuffledb/DBBackend.java | 30 ++++---
.../apache/spark/network/shuffledb/DBIterator.java | 22 +++--
.../apache/spark/network/shuffledb/LevelDB.java | 39 +++++++--
.../spark/network/shuffledb/LevelDBIterator.java | 93 ++++++++++++++++++++++
.../spark/network/shuffledb/StoreVersion.java | 57 +++++++++++++
.../org/apache/spark/network/util/DBProvider.java | 61 ++++++++++++++
.../apache/spark/network/util/LevelDBProvider.java | 45 +++--------
.../apache/spark/network/shuffle/Constants.java | 7 ++
.../shuffle/ExternalShuffleBlockResolver.java | 19 +++--
.../network/shuffle/RemoteBlockPushResolver.java | 21 +++--
.../spark/network/yarn/YarnShuffleService.java | 37 ++++++---
.../spark/deploy/ExternalShuffleService.scala | 10 ++-
.../org/apache/spark/internal/config/package.scala | 11 +++
.../yarn/YarnShuffleAlternateNameConfigSuite.scala | 9 ++-
.../deploy/yarn/YarnShuffleIntegrationSuite.scala | 31 ++++++--
.../network/shuffle/ShuffleTestAccessor.scala | 14 ++--
.../network/yarn/YarnShuffleServiceSuite.scala | 19 ++++-
18 files changed, 443 insertions(+), 119 deletions(-)
copy streaming/src/main/scala/org/apache/spark/streaming/receiver/ReceivedBlock.scala => common/network-common/src/main/java/org/apache/spark/network/shuffledb/DB.java (56%)
copy sql/catalyst/src/test/java/org/apache/spark/sql/streaming/JavaOutputModeSuite.java => common/network-common/src/main/java/org/apache/spark/network/shuffledb/DBBackend.java (60%)
copy core/src/main/java/org/apache/spark/api/java/function/DoubleFlatMapFunction.java => common/network-common/src/main/java/org/apache/spark/network/shuffledb/DBIterator.java (68%)
copy core/src/main/java/org/apache/spark/status/api/v1/ApplicationStatus.java => common/network-common/src/main/java/org/apache/spark/network/shuffledb/LevelDB.java (57%)
create mode 100644 common/network-common/src/main/java/org/apache/spark/network/shuffledb/LevelDBIterator.java
create mode 100644 common/network-common/src/main/java/org/apache/spark/network/shuffledb/StoreVersion.java
create mode 100644 common/network-common/src/main/java/org/apache/spark/network/util/DBProvider.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org