You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2019/05/23 20:16:17 UTC
[spark] branch master updated (1a68fc3 -> e9f3f62)
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git.
from 1a68fc3 [SPARK-27816][SQL] make TreeNode tag type safe
add e9f3f62 [SPARK-27677][CORE] Serve local disk persisted blocks by the external service after releasing executor by dynamic allocation
No new revisions were added by this update.
Summary of changes:
.../network/server/ChunkFetchRequestHandler.java | 3 +
.../network/server/OneForOneStreamManager.java | 5 +-
.../network/ChunkFetchRequestHandlerSuite.java | 42 ++--
.../network/TransportRequestHandlerSuite.java | 47 ++--
.../server/OneForOneStreamManagerSuite.java | 56 ++++-
.../apache/spark/network/shuffle/Constants.java | 14 +-
.../shuffle/ExternalShuffleBlockHandler.java | 77 +++++--
.../shuffle/ExternalShuffleBlockResolver.java | 79 +++++--
.../network/shuffle/ExternalShuffleClient.java | 42 +++-
.../spark/network/shuffle/ShuffleClient.java | 6 -
.../shuffle/protocol/BlockTransferMessage.java | 5 +-
.../{StreamHandle.java => BlocksRemoved.java} | 41 ++--
.../{OpenBlocks.java => RemoveBlocks.java} | 21 +-
.../CleanupNonShuffleServiceServedFilesSuite.java | 256 +++++++++++++++++++++
.../shuffle/ExternalShuffleBlockHandlerSuite.java | 63 +++--
.../shuffle/ExternalShuffleIntegrationSuite.java | 96 +++++++-
.../shuffle/NonShuffleFilesCleanupSuite.java | 221 ------------------
.../network/shuffle/TestShuffleDataContext.java | 34 ++-
.../apache/spark/ExecutorAllocationManager.scala | 12 +-
.../src/main/scala/org/apache/spark/SparkEnv.scala | 37 ++-
.../org/apache/spark/deploy/worker/Worker.scala | 7 +-
.../org/apache/spark/internal/config/package.scala | 13 +-
.../org/apache/spark/storage/BlockManager.scala | 65 +++---
.../apache/spark/storage/BlockManagerMaster.scala | 9 +-
.../spark/storage/BlockManagerMasterEndpoint.scala | 159 ++++++++++---
.../spark/storage/BlockManagerMessages.scala | 3 +-
.../org/apache/spark/storage/StorageUtils.scala | 19 +-
.../apache/spark/ExternalShuffleServiceSuite.scala | 43 +++-
.../spark/storage/BlockManagerInfoSuite.scala | 160 +++++++++++++
.../storage/BlockManagerReplicationSuite.scala | 6 +-
.../apache/spark/storage/BlockManagerSuite.scala | 73 +++++-
.../streaming/ReceivedBlockHandlerSuite.scala | 4 +-
32 files changed, 1237 insertions(+), 481 deletions(-)
copy mllib-local/src/main/scala/org/apache/spark/ml/impl/Utils.scala => common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/Constants.java (81%)
copy common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/{StreamHandle.java => BlocksRemoved.java} (57%)
copy common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/{OpenBlocks.java => RemoveBlocks.java} (84%)
create mode 100644 common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/CleanupNonShuffleServiceServedFilesSuite.java
delete mode 100644 common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/NonShuffleFilesCleanupSuite.java
create mode 100644 core/src/test/scala/org/apache/spark/storage/BlockManagerInfoSuite.scala
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org