You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2023/06/25 13:37:27 UTC
[incubator-celeborn] branch main updated: [CELEBORN-712] Make appUniqueId a member of ShuffleClientImpl and refactor code
This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 6b82ecdfa [CELEBORN-712] Make appUniqueId a member of ShuffleClientImpl and refactor code
6b82ecdfa is described below
commit 6b82ecdfa0869c76c8473023aabc37a48b2433d7
Author: zky.zhoukeyong <zk...@alibaba-inc.com>
AuthorDate: Sun Jun 25 21:37:16 2023 +0800
[CELEBORN-712] Make appUniqueId a member of ShuffleClientImpl and refactor code
### What changes were proposed in this pull request?
Make appUniqueId a member of ShuffleClientImpl and remove applicationId from RPC messages across client side, so it won't cause compatibility issues.
### Why are the changes needed?
Currently Celeborn Client is bound to a single application id, so there's no need to pass applicationId around in many RPC messages in client side.
### Does this PR introduce _any_ user-facing change?
In some logs the application id will not be printed, which should not be a problem.
### How was this patch tested?
UTs.
Closes #1621 from waitinfuture/appid.
Authored-by: zky.zhoukeyong <zk...@alibaba-inc.com>
Signed-off-by: zky.zhoukeyong <zk...@alibaba-inc.com>
---
.../plugin/flink/RemoteBufferStreamReader.java | 5 +-
.../flink/RemoteShuffleInputGateDelegation.java | 6 +-
.../celeborn/plugin/flink/RemoteShuffleMaster.java | 2 +-
.../plugin/flink/RemoteShuffleOutputGate.java | 29 +---
.../flink/readclient/FlinkShuffleClientImpl.java | 58 ++++---
.../plugin/flink/FlinkShuffleClientImplSuiteJ.java | 11 +-
.../flink/RemoteShuffleOutputGateSuiteJ.java | 11 +-
.../flink/RemoteShuffleResultPartitionSuiteJ.java | 3 +-
.../flink/RemoteShuffleResultPartitionSuiteJ.java | 3 +-
.../flink/RemoteShuffleResultPartitionSuiteJ.java | 3 +-
.../spark/shuffle/celeborn/SortBasedPusher.java | 2 -
.../shuffle/celeborn/HashBasedShuffleWriter.java | 11 +-
.../spark/shuffle/celeborn/RssShuffleManager.java | 9 +-
.../shuffle/celeborn/SortBasedShuffleWriter.java | 9 +-
.../spark/shuffle/celeborn/RssShuffleReader.scala | 2 +-
.../shuffle/celeborn/HashBasedShuffleWriter.java | 12 +-
.../spark/shuffle/celeborn/RssShuffleManager.java | 15 +-
.../shuffle/celeborn/SortBasedShuffleWriter.java | 9 +-
.../spark/shuffle/celeborn/RssShuffleReader.scala | 2 +-
.../org/apache/celeborn/client/ShuffleClient.java | 46 +++---
.../apache/celeborn/client/ShuffleClientImpl.java | 166 +++++++--------------
.../celeborn/client/write/DataPushQueue.java | 5 +-
.../apache/celeborn/client/write/DataPusher.java | 6 +-
.../celeborn/client/ChangePartitionManager.scala | 12 +-
.../org/apache/celeborn/client/CommitManager.scala | 8 +-
.../apache/celeborn/client/LifecycleManager.scala | 117 ++++++---------
.../celeborn/client/ReleasePartitionManager.scala | 4 +-
.../celeborn/client/commit/CommitHandler.scala | 6 +-
.../commit/ReducePartitionCommitHandler.scala | 4 +-
.../apache/celeborn/client/DummyShuffleClient.java | 32 ++--
.../celeborn/client/ShuffleClientBaseSuiteJ.java | 6 +-
.../celeborn/client/ShuffleClientSuiteJ.java | 7 +-
.../celeborn/client/write/DataPushQueueSuitJ.java | 2 -
.../celeborn/client/WithShuffleClientSuite.scala | 28 ++--
common/src/main/proto/TransportMessages.proto | 59 ++++----
.../common/protocol/message/ControlMessages.scala | 28 +---
.../org/apache/celeborn/common/util/Utils.scala | 4 +-
.../apache/celeborn/common/util/UtilsSuite.scala | 2 +-
.../celeborn/tests/flink/HeartbeatTest.scala | 3 +
.../celeborn/tests/client/ShuffleClientSuite.scala | 4 +-
.../celeborn/tests/spark/HeartbeatTest.scala | 6 +-
.../service/deploy/cluster/ReadWriteTestBase.scala | 16 +-
42 files changed, 292 insertions(+), 481 deletions(-)
diff --git a/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteBufferStreamReader.java b/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteBufferStreamReader.java
index 776ee6efc..b4b685b55 100644
--- a/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteBufferStreamReader.java
+++ b/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteBufferStreamReader.java
@@ -37,7 +37,6 @@ public class RemoteBufferStreamReader extends CreditListener {
private static Logger logger = LoggerFactory.getLogger(RemoteBufferStreamReader.class);
private TransferBufferPool bufferPool;
private FlinkShuffleClientImpl client;
- private String applicationId;
private int shuffleId;
private int partitionId;
private int subPartitionIndexStart;
@@ -53,14 +52,12 @@ public class RemoteBufferStreamReader extends CreditListener {
public RemoteBufferStreamReader(
FlinkShuffleClientImpl client,
ShuffleResourceDescriptor shuffleDescriptor,
- String applicationId,
int startSubIdx,
int endSubIdx,
TransferBufferPool bufferPool,
Consumer<ByteBuf> dataListener,
Consumer<Throwable> failureListener) {
this.client = client;
- this.applicationId = applicationId;
this.shuffleId = shuffleDescriptor.getShuffleId();
this.partitionId = shuffleDescriptor.getPartitionId();
this.bufferPool = bufferPool;
@@ -84,7 +81,7 @@ public class RemoteBufferStreamReader extends CreditListener {
try {
this.bufferStream =
client.readBufferedPartition(
- applicationId, shuffleId, partitionId, subPartitionIndexStart, subPartitionIndexEnd);
+ shuffleId, partitionId, subPartitionIndexStart, subPartitionIndexEnd);
bufferStream.open(
RemoteBufferStreamReader.this::requestBuffer, initialCredit, messageConsumer);
} catch (Exception e) {
diff --git a/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleInputGateDelegation.java b/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleInputGateDelegation.java
index 6f07ae924..823cad622 100644
--- a/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleInputGateDelegation.java
+++ b/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleInputGateDelegation.java
@@ -152,8 +152,12 @@ public class RemoteShuffleInputGateDelegation {
RemoteShuffleResource shuffleResource = remoteShuffleDescriptor.getShuffleResource();
try {
+ String appUniqueId =
+ ((RemoteShuffleDescriptor) (gateDescriptor.getShuffleDescriptors()[0]))
+ .getCelebornAppId();
this.shuffleClient =
FlinkShuffleClientImpl.get(
+ appUniqueId,
shuffleResource.getRssMetaServiceHost(),
shuffleResource.getRssMetaServicePort(),
shuffleResource.getRssMetaServiceTimestamp(),
@@ -191,13 +195,11 @@ public class RemoteShuffleInputGateDelegation {
remoteDescriptor.getShuffleResource().getMapPartitionShuffleDescriptor();
LOG.debug("create shuffle reader for descriptor {}", shuffleDescriptor);
- String applicationId = remoteDescriptor.getCelebornAppId();
RemoteBufferStreamReader reader =
new RemoteBufferStreamReader(
shuffleClient,
shuffleDescriptor,
- applicationId,
startSubIndex,
endSubIndex,
transferBufferPool,
diff --git a/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleMaster.java b/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleMaster.java
index 26b3ed63f..78881f0a5 100644
--- a/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleMaster.java
+++ b/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleMaster.java
@@ -100,7 +100,7 @@ public class RemoteShuffleMaster implements ShuffleMaster<RemoteShuffleDescripto
() -> {
try {
for (Integer shuffleId : shuffleIds) {
- lifecycleManager.handleUnregisterShuffle(celebornAppId, shuffleId);
+ lifecycleManager.handleUnregisterShuffle(shuffleId);
shuffleTaskInfo.removeExpiredShuffle(shuffleId);
}
shuffleResourceTracker.unRegisterJob(jobID);
diff --git a/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleOutputGate.java b/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleOutputGate.java
index 6ef65f836..f3e384e14 100644
--- a/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleOutputGate.java
+++ b/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleOutputGate.java
@@ -158,13 +158,7 @@ public class RemoteShuffleOutputGate {
newPartitionLoc =
flinkShuffleClient.regionStart(
- applicationId,
- shuffleId,
- mapId,
- attemptId,
- partitionLocation,
- currentRegionIndex,
- isBroadcast);
+ shuffleId, mapId, attemptId, partitionLocation, currentRegionIndex, isBroadcast);
// revived
if (newPartitionLoc.isPresent()) {
partitionLocation = newPartitionLoc.get();
@@ -172,13 +166,7 @@ public class RemoteShuffleOutputGate {
handshake(false);
// send regionstart again
flinkShuffleClient.regionStart(
- applicationId,
- shuffleId,
- mapId,
- attemptId,
- newPartitionLoc.get(),
- currentRegionIndex,
- isBroadcast);
+ shuffleId, mapId, attemptId, newPartitionLoc.get(), currentRegionIndex, isBroadcast);
}
} catch (IOException e) {
Utils.rethrowAsRuntimeException(e);
@@ -192,8 +180,7 @@ public class RemoteShuffleOutputGate {
public void regionFinish() throws InterruptedException {
bufferPacker.drain();
try {
- flinkShuffleClient.regionFinish(
- applicationId, shuffleId, mapId, attemptId, partitionLocation);
+ flinkShuffleClient.regionFinish(shuffleId, mapId, attemptId, partitionLocation);
currentRegionIndex++;
} catch (IOException e) {
Utils.rethrowAsRuntimeException(e);
@@ -203,7 +190,7 @@ public class RemoteShuffleOutputGate {
/** Indicates the writing/spilling is finished. */
public void finish() throws InterruptedException, IOException {
flinkShuffleClient.mapPartitionMapperEnd(
- applicationId, shuffleId, mapId, attemptId, numMappers, partitionLocation.getId());
+ shuffleId, mapId, attemptId, numMappers, partitionLocation.getId());
}
/** Close the transportation gate. */
@@ -212,7 +199,7 @@ public class RemoteShuffleOutputGate {
bufferPool.lazyDestroy();
}
bufferPacker.close();
- flinkShuffleClient.cleanup(applicationId, shuffleId, mapId, attemptId);
+ flinkShuffleClient.cleanup(shuffleId, mapId, attemptId);
}
/** Returns shuffle descriptor. */
@@ -224,6 +211,7 @@ public class RemoteShuffleOutputGate {
FlinkShuffleClientImpl getShuffleClient() {
try {
return FlinkShuffleClientImpl.get(
+ applicationId,
rssMetaServiceHost,
rssMetaServicePort,
rssMetaServiceTimestamp,
@@ -239,7 +227,6 @@ public class RemoteShuffleOutputGate {
public void write(ByteBuf byteBuf, int subIdx) {
try {
flinkShuffleClient.pushDataToLocation(
- applicationId,
shuffleId,
mapId,
attemptId,
@@ -256,14 +243,14 @@ public class RemoteShuffleOutputGate {
if (isFirstHandShake) {
partitionLocation =
flinkShuffleClient.registerMapPartitionTask(
- applicationId, shuffleId, numMappers, mapId, attemptId, partitionId);
+ shuffleId, numMappers, mapId, attemptId, partitionId);
Utils.checkNotNull(partitionLocation);
currentRegionIndex = 0;
}
try {
flinkShuffleClient.pushDataHandShake(
- applicationId, shuffleId, mapId, attemptId, numSubs, bufferSize, partitionLocation);
+ shuffleId, mapId, attemptId, numSubs, bufferSize, partitionLocation);
} catch (IOException e) {
Utils.rethrowAsRuntimeException(e);
}
diff --git a/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/readclient/FlinkShuffleClientImpl.java b/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/readclient/FlinkShuffleClientImpl.java
index 5d2d1a081..c1e93687f 100644
--- a/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/readclient/FlinkShuffleClientImpl.java
+++ b/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/readclient/FlinkShuffleClientImpl.java
@@ -71,6 +71,7 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
private long driverTimestamp;
public static FlinkShuffleClientImpl get(
+ String appUniqueId,
String driverHost,
int port,
long driverTimestamp,
@@ -81,12 +82,14 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
synchronized (FlinkShuffleClientImpl.class) {
if (null == _instance) {
_instance =
- new FlinkShuffleClientImpl(driverHost, port, driverTimestamp, conf, userIdentifier);
+ new FlinkShuffleClientImpl(
+ appUniqueId, driverHost, port, driverTimestamp, conf, userIdentifier);
initialized = true;
} else if (!initialized || _instance.driverTimestamp < driverTimestamp) {
_instance.shutdown();
_instance =
- new FlinkShuffleClientImpl(driverHost, port, driverTimestamp, conf, userIdentifier);
+ new FlinkShuffleClientImpl(
+ appUniqueId, driverHost, port, driverTimestamp, conf, userIdentifier);
initialized = true;
}
}
@@ -114,12 +117,13 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
}
public FlinkShuffleClientImpl(
+ String appUniqueId,
String driverHost,
int port,
long driverTimestamp,
CelebornConf conf,
UserIdentifier userIdentifier) {
- super(conf, userIdentifier);
+ super(appUniqueId, conf, userIdentifier);
String module = TransportModuleConstants.DATA_MODULE;
TransportConf dataTransportConf =
Utils.fromCelebornConf(conf, module, conf.getInt("celeborn." + module + ".io.threads", 8));
@@ -133,14 +137,10 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
}
public RssBufferStream readBufferedPartition(
- String applicationId,
- int shuffleId,
- int partitionId,
- int subPartitionIndexStart,
- int subPartitionIndexEnd)
+ int shuffleId, int partitionId, int subPartitionIndexStart, int subPartitionIndexEnd)
throws IOException {
- String shuffleKey = Utils.makeShuffleKey(applicationId, shuffleId);
- ReduceFileGroups fileGroups = loadFileGroup(applicationId, shuffleKey, shuffleId, partitionId);
+ String shuffleKey = Utils.makeShuffleKey(appUniqueId, shuffleId);
+ ReduceFileGroups fileGroups = loadFileGroup(shuffleId, partitionId);
if (fileGroups.partitionGroups.size() == 0
|| !fileGroups.partitionGroups.containsKey(partitionId)) {
logger.error("Shuffle data is empty for shuffle {} partitionId {}.", shuffleId, partitionId);
@@ -158,29 +158,26 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
}
@Override
- protected ReduceFileGroups updateFileGroup(
- String applicationId, String shuffleKey, int shuffleId, int partitionId) throws IOException {
+ protected ReduceFileGroups updateFileGroup(int shuffleId, int partitionId) throws IOException {
ReduceFileGroups reduceFileGroups =
reduceFileGroupsMap.computeIfAbsent(shuffleId, (id) -> new ReduceFileGroups());
if (reduceFileGroups.partitionIds != null
&& reduceFileGroups.partitionIds.contains(partitionId)) {
logger.debug(
- "use cached file groups for partition: {}",
- Utils.makeReducerKey(applicationId, shuffleId, partitionId));
+ "use cached file groups for partition: {}", Utils.makeReducerKey(shuffleId, partitionId));
} else {
synchronized (reduceFileGroups) {
if (reduceFileGroups.partitionIds != null
&& reduceFileGroups.partitionIds.contains(partitionId)) {
logger.debug(
"use cached file groups for partition: {}",
- Utils.makeReducerKey(applicationId, shuffleId, partitionId));
+ Utils.makeReducerKey(shuffleId, partitionId));
} else {
// refresh file groups
- ReduceFileGroups newGroups = loadFileGroupInternal(applicationId, shuffleKey, shuffleId);
+ ReduceFileGroups newGroups = loadFileGroupInternal(shuffleId);
if (newGroups == null || !newGroups.partitionIds.contains(partitionId)) {
throw new IOException(
- "shuffle data lost for partition: "
- + Utils.makeReducerKey(applicationId, shuffleId, partitionId));
+ "shuffle data lost for partition: " + Utils.makeReducerKey(shuffleId, partitionId));
}
reduceFileGroups.update(newGroups);
}
@@ -194,7 +191,6 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
}
public int pushDataToLocation(
- String applicationId,
int shuffleId,
int mapId,
int attemptId,
@@ -205,7 +201,6 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
throws IOException {
// mapKey
final String mapKey = Utils.makeMapKey(shuffleId, mapId, attemptId);
- final String shuffleKey = Utils.makeShuffleKey(applicationId, shuffleId);
PushState pushState = getPushState(mapKey);
@@ -222,7 +217,7 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
logger.debug(
"Do push data byteBuf size {} for app {} shuffle {} map {} attempt {} reduce {} batch {}.",
totalLength,
- applicationId,
+ appUniqueId,
shuffleId,
mapId,
attemptId,
@@ -236,6 +231,7 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
// build PushData request
NettyManagedBuffer buffer = new NettyManagedBuffer(data);
+ final String shuffleKey = Utils.makeShuffleKey(appUniqueId, shuffleId);
PushData pushData = new PushData(MASTER_MODE, shuffleKey, location.getUniqueId(), buffer);
// build callback
@@ -304,7 +300,6 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
}
public void pushDataHandShake(
- String applicationId,
int shuffleId,
int mapId,
int attemptId,
@@ -321,13 +316,13 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
location,
pushState,
() -> {
- String shuffleKey = Utils.makeShuffleKey(applicationId, shuffleId);
+ String shuffleKey = Utils.makeShuffleKey(appUniqueId, shuffleId);
logger.info(
"PushDataHandShake shuffleKey {} attemptId {} locationId {}",
shuffleKey,
attemptId,
location.getUniqueId());
- logger.debug("PushDataHandShake location {}", location.toString());
+ logger.debug("PushDataHandShake location {}", location);
TransportClient client = createClientWaitingInFlightRequest(location, mapKey, pushState);
PushDataHandShake handShake =
new PushDataHandShake(
@@ -343,7 +338,6 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
}
public Optional<PartitionLocation> regionStart(
- String applicationId,
int shuffleId,
int mapId,
int attemptId,
@@ -360,7 +354,7 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
location,
pushState,
() -> {
- String shuffleKey = Utils.makeShuffleKey(applicationId, shuffleId);
+ String shuffleKey = Utils.makeShuffleKey(appUniqueId, shuffleId);
logger.info(
"RegionStart for shuffle {} regionId {} attemptId {} locationId {}.",
shuffleId,
@@ -385,7 +379,6 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
PbChangeLocationResponse response =
driverRssMetaService.askSync(
ControlMessages.Revive$.MODULE$.apply(
- applicationId,
shuffleId,
mapId,
attemptId,
@@ -415,8 +408,7 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
});
}
- public void regionFinish(
- String applicationId, int shuffleId, int mapId, int attemptId, PartitionLocation location)
+ public void regionFinish(int shuffleId, int mapId, int attemptId, PartitionLocation location)
throws IOException {
final String mapKey = Utils.makeMapKey(shuffleId, mapId, attemptId);
final PushState pushState = pushStates.computeIfAbsent(mapKey, (s) -> new PushState(conf));
@@ -427,14 +419,14 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
location,
pushState,
() -> {
- final String shuffleKey = Utils.makeShuffleKey(applicationId, shuffleId);
+ final String shuffleKey = Utils.makeShuffleKey(appUniqueId, shuffleId);
logger.info(
"RegionFinish for shuffle {} map {} attemptId {} locationId {}.",
shuffleId,
mapId,
attemptId,
location.getUniqueId());
- logger.debug("RegionFinish for location {}.", location.toString());
+ logger.debug("RegionFinish for location {}.", location);
TransportClient client = createClientWaitingInFlightRequest(location, mapKey, pushState);
RegionFinish regionFinish =
new RegionFinish(MASTER_MODE, shuffleKey, location.getUniqueId(), attemptId);
@@ -525,9 +517,9 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl {
}
@Override
- public void cleanup(String applicationId, int shuffleId, int mapId, int attemptId) {
+ public void cleanup(int shuffleId, int mapId, int attemptId) {
final String mapKey = Utils.makeMapKey(shuffleId, mapId, attemptId);
- super.cleanup(applicationId, shuffleId, mapId, attemptId);
+ super.cleanup(shuffleId, mapId, attemptId);
if (currentClient != null) {
currentClient.remove(mapKey);
}
diff --git a/client-flink/common/src/test/java/org/apache/celeborn/plugin/flink/FlinkShuffleClientImplSuiteJ.java b/client-flink/common/src/test/java/org/apache/celeborn/plugin/flink/FlinkShuffleClientImplSuiteJ.java
index ec495b12e..827df2a69 100644
--- a/client-flink/common/src/test/java/org/apache/celeborn/plugin/flink/FlinkShuffleClientImplSuiteJ.java
+++ b/client-flink/common/src/test/java/org/apache/celeborn/plugin/flink/FlinkShuffleClientImplSuiteJ.java
@@ -54,7 +54,8 @@ public class FlinkShuffleClientImplSuiteJ {
public void setup() throws IOException, InterruptedException {
conf = new CelebornConf();
shuffleClient =
- new FlinkShuffleClientImpl("localhost", 1232, System.currentTimeMillis(), conf, null) {
+ new FlinkShuffleClientImpl(
+ "APP", "localhost", 1232, System.currentTimeMillis(), conf, null) {
@Override
public void setupMetaServiceRef(String host, int port) {}
};
@@ -86,7 +87,7 @@ public class FlinkShuffleClientImplSuiteJ {
});
int pushDataLen =
- shuffleClient.pushDataToLocation("1", 2, 3, 4, 5, byteBuf, masterLocation, () -> {});
+ shuffleClient.pushDataToLocation(2, 3, 4, 5, byteBuf, masterLocation, () -> {});
Assert.assertEquals(BufferSize, pushDataLen);
}
@@ -103,7 +104,7 @@ public class FlinkShuffleClientImplSuiteJ {
return mockedFuture;
});
int pushDataLen =
- shuffleClient.pushDataToLocation("1", 2, 3, 4, 5, byteBuf, masterLocation, () -> {});
+ shuffleClient.pushDataToLocation(2, 3, 4, 5, byteBuf, masterLocation, () -> {});
}
@Test
@@ -117,12 +118,12 @@ public class FlinkShuffleClientImplSuiteJ {
return mockedFuture;
});
// first push just set pushdata.exception
- shuffleClient.pushDataToLocation("1", 2, 3, 4, 5, byteBuf, masterLocation, () -> {});
+ shuffleClient.pushDataToLocation(2, 3, 4, 5, byteBuf, masterLocation, () -> {});
boolean isFailed = false;
// second push will throw exception
try {
- shuffleClient.pushDataToLocation("1", 2, 3, 4, 5, byteBuf, masterLocation, () -> {});
+ shuffleClient.pushDataToLocation(2, 3, 4, 5, byteBuf, masterLocation, () -> {});
} catch (IOException e) {
isFailed = true;
} finally {
diff --git a/client-flink/common/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleOutputGateSuiteJ.java b/client-flink/common/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleOutputGateSuiteJ.java
index 4d74bd492..b5420a2f5 100644
--- a/client-flink/common/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleOutputGateSuiteJ.java
+++ b/client-flink/common/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleOutputGateSuiteJ.java
@@ -53,17 +53,16 @@ public class RemoteShuffleOutputGateSuiteJ {
PartitionLocation partitionLocation =
new PartitionLocation(1, 0, "localhost", 123, 245, 789, 238, PartitionLocation.Mode.MASTER);
- when(shuffleClient.registerMapPartitionTask(
- any(), anyInt(), anyInt(), anyInt(), anyInt(), anyInt()))
+ when(shuffleClient.registerMapPartitionTask(anyInt(), anyInt(), anyInt(), anyInt(), anyInt()))
.thenAnswer(t -> partitionLocation);
doNothing()
.when(remoteShuffleOutputGate.flinkShuffleClient)
- .pushDataHandShake(anyString(), anyInt(), anyInt(), anyInt(), anyInt(), anyInt(), any());
+ .pushDataHandShake(anyInt(), anyInt(), anyInt(), anyInt(), anyInt(), any());
remoteShuffleOutputGate.handshake(true);
when(remoteShuffleOutputGate.flinkShuffleClient.regionStart(
- any(), anyInt(), anyInt(), anyInt(), any(), anyInt(), anyBoolean()))
+ anyInt(), anyInt(), anyInt(), any(), anyInt(), anyBoolean()))
.thenAnswer(t -> Optional.empty());
remoteShuffleOutputGate.regionStart(false);
@@ -71,12 +70,12 @@ public class RemoteShuffleOutputGateSuiteJ {
doNothing()
.when(remoteShuffleOutputGate.flinkShuffleClient)
- .regionFinish(any(), anyInt(), anyInt(), anyInt(), any());
+ .regionFinish(anyInt(), anyInt(), anyInt(), any());
remoteShuffleOutputGate.regionFinish();
doNothing()
.when(remoteShuffleOutputGate.flinkShuffleClient)
- .mapperEnd(any(), anyInt(), anyInt(), anyInt(), anyInt());
+ .mapperEnd(anyInt(), anyInt(), anyInt(), anyInt());
remoteShuffleOutputGate.finish();
doNothing().when(remoteShuffleOutputGate.flinkShuffleClient).shutdown();
diff --git a/client-flink/flink-1.14/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleResultPartitionSuiteJ.java b/client-flink/flink-1.14/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleResultPartitionSuiteJ.java
index de3190cec..044b7e7dd 100644
--- a/client-flink/flink-1.14/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleResultPartitionSuiteJ.java
+++ b/client-flink/flink-1.14/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleResultPartitionSuiteJ.java
@@ -21,7 +21,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -455,7 +454,7 @@ public class RemoteShuffleResultPartitionSuiteJ {
@Override
FlinkShuffleClientImpl getShuffleClient() {
FlinkShuffleClientImpl client = mock(FlinkShuffleClientImpl.class);
- doNothing().when(client).cleanup(anyString(), anyInt(), anyInt(), anyInt());
+ doNothing().when(client).cleanup(anyInt(), anyInt(), anyInt());
return client;
}
diff --git a/client-flink/flink-1.15/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleResultPartitionSuiteJ.java b/client-flink/flink-1.15/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleResultPartitionSuiteJ.java
index de3190cec..044b7e7dd 100644
--- a/client-flink/flink-1.15/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleResultPartitionSuiteJ.java
+++ b/client-flink/flink-1.15/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleResultPartitionSuiteJ.java
@@ -21,7 +21,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -455,7 +454,7 @@ public class RemoteShuffleResultPartitionSuiteJ {
@Override
FlinkShuffleClientImpl getShuffleClient() {
FlinkShuffleClientImpl client = mock(FlinkShuffleClientImpl.class);
- doNothing().when(client).cleanup(anyString(), anyInt(), anyInt(), anyInt());
+ doNothing().when(client).cleanup(anyInt(), anyInt(), anyInt());
return client;
}
diff --git a/client-flink/flink-1.17/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleResultPartitionSuiteJ.java b/client-flink/flink-1.17/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleResultPartitionSuiteJ.java
index de3190cec..044b7e7dd 100644
--- a/client-flink/flink-1.17/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleResultPartitionSuiteJ.java
+++ b/client-flink/flink-1.17/src/test/java/org/apache/celeborn/plugin/flink/RemoteShuffleResultPartitionSuiteJ.java
@@ -21,7 +21,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -455,7 +454,7 @@ public class RemoteShuffleResultPartitionSuiteJ {
@Override
FlinkShuffleClientImpl getShuffleClient() {
FlinkShuffleClientImpl client = mock(FlinkShuffleClientImpl.class);
- doNothing().when(client).cleanup(anyString(), anyInt(), anyInt(), anyInt());
+ doNothing().when(client).cleanup(anyInt(), anyInt(), anyInt());
return client;
}
diff --git a/client-spark/common/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedPusher.java b/client-spark/common/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedPusher.java
index 80bb6faa9..573a02b19 100644
--- a/client-spark/common/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedPusher.java
+++ b/client-spark/common/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedPusher.java
@@ -117,7 +117,6 @@ public class SortBasedPusher extends MemoryConsumer {
try {
dataPusher =
new DataPusher(
- appId,
shuffleId,
mapId,
attemptNumber,
@@ -162,7 +161,6 @@ public class SortBasedPusher extends MemoryConsumer {
} else {
int bytesWritten =
rssShuffleClient.mergeData(
- appId,
shuffleId,
mapId,
attemptNumber,
diff --git a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java
index e2f12f969..767819a1e 100644
--- a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java
+++ b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java
@@ -64,7 +64,6 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
private final ShuffleDependency<K, V, C> dep;
private final Partitioner partitioner;
private final ShuffleWriteMetrics writeMetrics;
- private final String appId;
private final int shuffleId;
private final int mapId;
private final TaskContext taskContext;
@@ -109,7 +108,6 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
throws IOException {
this.mapId = mapId;
this.dep = handle.dependency();
- this.appId = handle.appUniqueId();
this.shuffleId = dep.shuffleId();
SerializerInstance serializer = dep.serializer().newInstance();
this.partitioner = dep.partitioner();
@@ -142,7 +140,6 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
try {
dataPusher =
new DataPusher(
- appId,
shuffleId,
mapId,
taskContext.attemptNumber(),
@@ -277,7 +274,6 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
logger.debug("Push giant record for partition {}, size {}.", partitionId, numBytes);
int bytesWritten =
rssShuffleClient.pushData(
- appId,
shuffleId,
mapId,
taskContext.attemptNumber(),
@@ -332,7 +328,6 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
if (size > 0) {
int bytesWritten =
rssShuffleClient.mergeData(
- appId,
shuffleId,
mapId,
taskContext.attemptNumber(),
@@ -348,7 +343,7 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
writeMetrics.incBytesWritten(bytesWritten);
}
}
- rssShuffleClient.pushMergedData(appId, shuffleId, mapId, taskContext.attemptNumber());
+ rssShuffleClient.pushMergedData(shuffleId, mapId, taskContext.attemptNumber());
updateMapStatus();
@@ -357,7 +352,7 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
sendOffsets = null;
long waitStartTime = System.nanoTime();
- rssShuffleClient.mapperEnd(appId, shuffleId, mapId, taskContext.attemptNumber(), numMappers);
+ rssShuffleClient.mapperEnd(shuffleId, mapId, taskContext.attemptNumber(), numMappers);
writeMetrics.incWriteTime(System.nanoTime() - waitStartTime);
BlockManagerId bmId = SparkEnv.get().blockManager().shuffleServerId();
@@ -394,7 +389,7 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
}
}
} finally {
- rssShuffleClient.cleanup(appId, shuffleId, mapId, taskContext.attemptNumber());
+ rssShuffleClient.cleanup(shuffleId, mapId, taskContext.attemptNumber());
}
}
}
diff --git a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
index 9049cc3ce..6b7ef80e9 100644
--- a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
+++ b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
@@ -103,6 +103,7 @@ public class RssShuffleManager implements ShuffleManager {
lifecycleManager = new LifecycleManager(appId, celebornConf);
rssShuffleClient =
ShuffleClient.get(
+ appUniqueId,
lifecycleManager.getRssMetaServiceHost(),
lifecycleManager.getRssMetaServicePort(),
celebornConf,
@@ -149,7 +150,7 @@ public class RssShuffleManager implements ShuffleManager {
if (rssShuffleClient == null) {
return false;
}
- return rssShuffleClient.unregisterShuffle(appUniqueId, shuffleId, isDriver());
+ return rssShuffleClient.unregisterShuffle(shuffleId, isDriver());
}
@Override
@@ -179,7 +180,11 @@ public class RssShuffleManager implements ShuffleManager {
RssShuffleHandle<K, V, ?> h = ((RssShuffleHandle<K, V, ?>) handle);
ShuffleClient client =
ShuffleClient.get(
- h.rssMetaServiceHost(), h.rssMetaServicePort(), celebornConf, h.userIdentifier());
+ appUniqueId,
+ h.rssMetaServiceHost(),
+ h.rssMetaServicePort(),
+ celebornConf,
+ h.userIdentifier());
if (ShuffleMode.SORT.equals(celebornConf.shuffleWriterMode())) {
ExecutorService pushThread =
celebornConf.clientPushSortPipelineEnabled() ? getPusherThread() : null;
diff --git a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java
index 33292f3a7..e4f587cca 100644
--- a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java
+++ b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java
@@ -61,7 +61,6 @@ public class SortBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
private final ShuffleDependency<K, V, C> dep;
private final Partitioner partitioner;
private final ShuffleWriteMetrics writeMetrics;
- private final String appId;
private final int shuffleId;
private final int mapId;
private final TaskContext taskContext;
@@ -105,7 +104,6 @@ public class SortBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
throws IOException {
this.mapId = taskContext.partitionId();
this.dep = dep;
- this.appId = appId;
this.shuffleId = dep.shuffleId();
SerializerInstance serializer = dep.serializer().newInstance();
this.partitioner = dep.partitioner();
@@ -304,7 +302,6 @@ public class SortBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
logger.debug("Push giant record, size {}.", Utils.bytesToString(numBytes));
int bytesWritten =
rssShuffleClient.pushData(
- appId,
shuffleId,
mapId,
taskContext.attemptNumber(),
@@ -338,12 +335,12 @@ public class SortBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
}
writeMetrics.incWriteTime(System.nanoTime() - pushStartTime);
- rssShuffleClient.pushMergedData(appId, shuffleId, mapId, taskContext.attemptNumber());
+ rssShuffleClient.pushMergedData(shuffleId, mapId, taskContext.attemptNumber());
updateMapStatus();
long waitStartTime = System.nanoTime();
- rssShuffleClient.mapperEnd(appId, shuffleId, mapId, taskContext.attemptNumber(), numMappers);
+ rssShuffleClient.mapperEnd(shuffleId, mapId, taskContext.attemptNumber(), numMappers);
writeMetrics.incWriteTime(System.nanoTime() - waitStartTime);
}
@@ -379,7 +376,7 @@ public class SortBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
} catch (IOException e) {
return Option.apply(null);
} finally {
- rssShuffleClient.cleanup(appId, shuffleId, mapId, taskContext.attemptNumber());
+ rssShuffleClient.cleanup(shuffleId, mapId, taskContext.attemptNumber());
}
}
diff --git a/client-spark/spark-2/src/main/scala/org/apache/spark/shuffle/celeborn/RssShuffleReader.scala b/client-spark/spark-2/src/main/scala/org/apache/spark/shuffle/celeborn/RssShuffleReader.scala
index 3d3e0e700..55a646d04 100644
--- a/client-spark/spark-2/src/main/scala/org/apache/spark/shuffle/celeborn/RssShuffleReader.scala
+++ b/client-spark/spark-2/src/main/scala/org/apache/spark/shuffle/celeborn/RssShuffleReader.scala
@@ -40,6 +40,7 @@ class RssShuffleReader[K, C](
private val dep = handle.dependency
private val essShuffleClient = ShuffleClient.get(
+ handle.appUniqueId,
handle.rssMetaServiceHost,
handle.rssMetaServicePort,
conf,
@@ -63,7 +64,6 @@ class RssShuffleReader[K, C](
if (handle.numMaps > 0) {
val start = System.currentTimeMillis()
val inputStream = essShuffleClient.readPartition(
- handle.appUniqueId,
handle.shuffleId,
partitionId,
context.attemptNumber(),
diff --git a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java
index 34c2d9937..f5b62a030 100644
--- a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java
+++ b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java
@@ -67,7 +67,6 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
private final ShuffleDependency<K, V, C> dep;
private final Partitioner partitioner;
private final ShuffleWriteMetricsReporter writeMetrics;
- private final String appId;
private final int shuffleId;
private final int mapId;
private final TaskContext taskContext;
@@ -130,7 +129,6 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
throws IOException {
this.mapId = taskContext.partitionId();
this.dep = handle.dependency();
- this.appId = handle.appUniqueId();
this.shuffleId = dep.shuffleId();
SerializerInstance serializer = dep.serializer().newInstance();
this.partitioner = dep.partitioner();
@@ -161,7 +159,6 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
try {
dataPusher =
new DataPusher(
- appId,
shuffleId,
mapId,
taskContext.attemptNumber(),
@@ -357,7 +354,6 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
logger.debug("Push giant record, size {}.", numBytes);
int bytesWritten =
rssShuffleClient.pushData(
- appId,
shuffleId,
mapId,
taskContext.attemptNumber(),
@@ -410,7 +406,6 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
}
int bytesWritten =
rssShuffleClient.mergeData(
- appId,
shuffleId,
mapId,
taskContext.attemptNumber(),
@@ -437,7 +432,6 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
if (size > 0) {
int bytesWritten =
rssShuffleClient.mergeData(
- appId,
shuffleId,
mapId,
taskContext.attemptNumber(),
@@ -468,13 +462,13 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
} else {
closeRowWrite();
}
- rssShuffleClient.pushMergedData(appId, shuffleId, mapId, taskContext.attemptNumber());
+ rssShuffleClient.pushMergedData(shuffleId, mapId, taskContext.attemptNumber());
writeMetrics.incWriteTime(System.nanoTime() - pushMergedDataTime);
updateMapStatus();
long waitStartTime = System.nanoTime();
- rssShuffleClient.mapperEnd(appId, shuffleId, mapId, taskContext.attemptNumber(), numMappers);
+ rssShuffleClient.mapperEnd(shuffleId, mapId, taskContext.attemptNumber(), numMappers);
writeMetrics.incWriteTime(System.nanoTime() - waitStartTime);
BlockManagerId bmId = SparkEnv.get().blockManager().shuffleServerId();
@@ -511,7 +505,7 @@ public class HashBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
}
}
} finally {
- rssShuffleClient.cleanup(appId, shuffleId, mapId, taskContext.attemptNumber());
+ rssShuffleClient.cleanup(shuffleId, mapId, taskContext.attemptNumber());
}
}
diff --git a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
index 9fbba3b4a..154903f70 100644
--- a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
+++ b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
@@ -90,7 +90,7 @@ public class RssShuffleManager implements ShuffleManager {
return _sortShuffleManager;
}
- private void initializeLifecycleManager(String appId) {
+ private void initializeLifecycleManager() {
// Only create LifecycleManager singleton in Driver. When register shuffle multiple times, we
// need to ensure that LifecycleManager will only be created once. Parallelism needs to be
// considered in this place, because if there is one RDD that depends on multiple RDDs
@@ -98,9 +98,10 @@ public class RssShuffleManager implements ShuffleManager {
if (isDriver() && lifecycleManager == null) {
synchronized (this) {
if (lifecycleManager == null) {
- lifecycleManager = new LifecycleManager(appId, celebornConf);
+ lifecycleManager = new LifecycleManager(appUniqueId, celebornConf);
rssShuffleClient =
ShuffleClient.get(
+ appUniqueId,
lifecycleManager.getRssMetaServiceHost(),
lifecycleManager.getRssMetaServicePort(),
celebornConf,
@@ -117,7 +118,7 @@ public class RssShuffleManager implements ShuffleManager {
// is the same SparkContext among different shuffleIds.
// This method may be called many times.
appUniqueId = SparkUtils.appUniqueId(dependency.rdd().context());
- initializeLifecycleManager(appUniqueId);
+ initializeLifecycleManager();
if (fallbackPolicyRunner.applyAllFallbackPolicy(
lifecycleManager, dependency.partitioner().numPartitions())) {
@@ -147,7 +148,7 @@ public class RssShuffleManager implements ShuffleManager {
if (rssShuffleClient == null) {
return false;
}
- return rssShuffleClient.unregisterShuffle(appUniqueId, shuffleId, isDriver());
+ return rssShuffleClient.unregisterShuffle(shuffleId, isDriver());
}
@Override
@@ -181,7 +182,11 @@ public class RssShuffleManager implements ShuffleManager {
RssShuffleHandle<K, V, ?> h = ((RssShuffleHandle<K, V, ?>) handle);
ShuffleClient client =
ShuffleClient.get(
- h.rssMetaServiceHost(), h.rssMetaServicePort(), celebornConf, h.userIdentifier());
+ h.appUniqueId(),
+ h.rssMetaServiceHost(),
+ h.rssMetaServicePort(),
+ celebornConf,
+ h.userIdentifier());
if (ShuffleMode.SORT.equals(celebornConf.shuffleWriterMode())) {
ExecutorService pushThread =
celebornConf.clientPushSortPipelineEnabled() ? getPusherThread() : null;
diff --git a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java
index a62e2c21a..b4ff196fd 100644
--- a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java
+++ b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java
@@ -61,7 +61,6 @@ public class SortBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
private final ShuffleDependency<K, V, C> dep;
private final Partitioner partitioner;
private final ShuffleWriteMetricsReporter writeMetrics;
- private final String appId;
private final int shuffleId;
private final int mapId;
private final TaskContext taskContext;
@@ -107,7 +106,6 @@ public class SortBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
throws IOException {
this.mapId = taskContext.partitionId();
this.dep = dep;
- this.appId = appId;
this.shuffleId = dep.shuffleId();
SerializerInstance serializer = dep.serializer().newInstance();
this.partitioner = dep.partitioner();
@@ -327,7 +325,6 @@ public class SortBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
logger.debug("Push giant record, size {}.", Utils.bytesToString(numBytes));
int bytesWritten =
rssShuffleClient.pushData(
- appId,
shuffleId,
mapId,
taskContext.attemptNumber(),
@@ -360,13 +357,13 @@ public class SortBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
currentPusher.close();
}
- rssShuffleClient.pushMergedData(appId, shuffleId, mapId, taskContext.attemptNumber());
+ rssShuffleClient.pushMergedData(shuffleId, mapId, taskContext.attemptNumber());
writeMetrics.incWriteTime(System.nanoTime() - pushStartTime);
updateMapStatus();
long waitStartTime = System.nanoTime();
- rssShuffleClient.mapperEnd(appId, shuffleId, mapId, taskContext.attemptNumber(), numMappers);
+ rssShuffleClient.mapperEnd(shuffleId, mapId, taskContext.attemptNumber(), numMappers);
writeMetrics.incWriteTime(System.nanoTime() - waitStartTime);
}
@@ -400,7 +397,7 @@ public class SortBasedShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
}
}
} finally {
- rssShuffleClient.cleanup(appId, shuffleId, mapId, taskContext.attemptNumber());
+ rssShuffleClient.cleanup(shuffleId, mapId, taskContext.attemptNumber());
}
}
diff --git a/client-spark/spark-3/src/main/scala/org/apache/spark/shuffle/celeborn/RssShuffleReader.scala b/client-spark/spark-3/src/main/scala/org/apache/spark/shuffle/celeborn/RssShuffleReader.scala
index e0cc31a06..03daac278 100644
--- a/client-spark/spark-3/src/main/scala/org/apache/spark/shuffle/celeborn/RssShuffleReader.scala
+++ b/client-spark/spark-3/src/main/scala/org/apache/spark/shuffle/celeborn/RssShuffleReader.scala
@@ -42,6 +42,7 @@ class RssShuffleReader[K, C](
private val dep = handle.dependency
private val rssShuffleClient = ShuffleClient.get(
+ handle.appUniqueId,
handle.rssMetaServiceHost,
handle.rssMetaServicePort,
conf,
@@ -80,7 +81,6 @@ class RssShuffleReader[K, C](
if (handle.numMappers > 0) {
val start = System.currentTimeMillis()
val inputStream = rssShuffleClient.readPartition(
- handle.appUniqueId,
handle.shuffleId,
partitionId,
context.attemptNumber(),
diff --git a/client/src/main/java/org/apache/celeborn/client/ShuffleClient.java b/client/src/main/java/org/apache/celeborn/client/ShuffleClient.java
index abd3399ba..2ec83dc82 100644
--- a/client/src/main/java/org/apache/celeborn/client/ShuffleClient.java
+++ b/client/src/main/java/org/apache/celeborn/client/ShuffleClient.java
@@ -52,7 +52,11 @@ public abstract class ShuffleClient {
protected ShuffleClient() {}
public static ShuffleClient get(
- String driverHost, int port, CelebornConf conf, UserIdentifier userIdentifier) {
+ String appUniqueId,
+ String driverHost,
+ int port,
+ CelebornConf conf,
+ UserIdentifier userIdentifier) {
if (null == _instance || !initialized) {
synchronized (ShuffleClient.class) {
if (null == _instance) {
@@ -61,12 +65,12 @@ public abstract class ShuffleClient {
// ShuffleClient is building a singleton, it may cause the MetaServiceEndpoint to not be
// assigned. An Executor will only construct a ShuffleClient singleton once. At this time,
// when communicating with MetaService, it will cause a NullPointerException.
- _instance = new ShuffleClientImpl(conf, userIdentifier);
+ _instance = new ShuffleClientImpl(appUniqueId, conf, userIdentifier);
_instance.setupMetaServiceRef(driverHost, port);
initialized = true;
} else if (!initialized) {
_instance.shutdown();
- _instance = new ShuffleClientImpl(conf, userIdentifier);
+ _instance = new ShuffleClientImpl(appUniqueId, conf, userIdentifier);
_instance.setupMetaServiceRef(driverHost, port);
initialized = true;
}
@@ -104,7 +108,6 @@ public abstract class ShuffleClient {
// Write data to a specific reduce partition
public abstract int pushData(
- String applicationId,
int shuffleId,
int mapId,
int attemptId,
@@ -120,7 +123,6 @@ public abstract class ShuffleClient {
throws IOException;
public abstract int mergeData(
- String applicationId,
int shuffleId,
int mapId,
int attemptId,
@@ -132,51 +134,37 @@ public abstract class ShuffleClient {
int numPartitions)
throws IOException;
- public abstract void pushMergedData(String applicationId, int shuffleId, int mapId, int attemptId)
- throws IOException;
+ public abstract void pushMergedData(int shuffleId, int mapId, int attemptId) throws IOException;
// Report partition locations written by the completed map task of ReducePartition Shuffle Type
- public abstract void mapperEnd(
- String applicationId, int shuffleId, int mapId, int attemptId, int numMappers)
+ public abstract void mapperEnd(int shuffleId, int mapId, int attemptId, int numMappers)
throws IOException;
// Report partition locations written by the completed map task of MapPartition Shuffle Type
public abstract void mapPartitionMapperEnd(
- String applicationId,
- int shuffleId,
- int mapId,
- int attemptId,
- int numMappers,
- int partitionId)
- throws IOException;
+ int shuffleId, int mapId, int attemptId, int numMappers, int partitionId) throws IOException;
// Cleanup states of the map task
- public abstract void cleanup(String applicationId, int shuffleId, int mapId, int attemptId);
+ public abstract void cleanup(int shuffleId, int mapId, int attemptId);
// Reduce side read partition which is deduplicated by mapperId+mapperAttemptNum+batchId, batchId
// is a self-incrementing variable hidden in the implementation when sending data.
public abstract RssInputStream readPartition(
- String applicationId,
- int shuffleId,
- int partitionId,
- int attemptNumber,
- int startMapIndex,
- int endMapIndex)
+ int shuffleId, int partitionId, int attemptNumber, int startMapIndex, int endMapIndex)
throws IOException;
- public abstract RssInputStream readPartition(
- String applicationId, int shuffleId, int partitionId, int attemptNumber) throws IOException;
+ public abstract RssInputStream readPartition(int shuffleId, int partitionId, int attemptNumber)
+ throws IOException;
- public abstract boolean unregisterShuffle(String applicationId, int shuffleId, boolean isDriver);
+ public abstract boolean unregisterShuffle(int shuffleId, boolean isDriver);
public abstract void shutdown();
public abstract PartitionLocation registerMapPartitionTask(
- String appId, int shuffleId, int numMappers, int mapId, int attemptId, int partitionId)
- throws IOException;
+ int shuffleId, int numMappers, int mapId, int attemptId, int partitionId) throws IOException;
public abstract ConcurrentHashMap<Integer, PartitionLocation> getPartitionLocation(
- String applicationId, int shuffleId, int numMappers, int numPartitions);
+ int shuffleId, int numMappers, int numPartitions);
public abstract PushState getPushState(String mapKey);
}
diff --git a/client/src/main/java/org/apache/celeborn/client/ShuffleClientImpl.java b/client/src/main/java/org/apache/celeborn/client/ShuffleClientImpl.java
index e7a57400b..6edf24a04 100644
--- a/client/src/main/java/org/apache/celeborn/client/ShuffleClientImpl.java
+++ b/client/src/main/java/org/apache/celeborn/client/ShuffleClientImpl.java
@@ -109,6 +109,8 @@ public class ShuffleClientImpl extends ShuffleClient {
private final ExecutorService partitionSplitPool;
private final Map<Integer, Set<Integer>> splitting = JavaUtils.newConcurrentHashMap();
+ protected final String appUniqueId;
+
ThreadLocal<Compressor> compressorThreadLocal =
new ThreadLocal<Compressor>() {
@Override
@@ -148,8 +150,9 @@ public class ShuffleClientImpl extends ShuffleClient {
protected final Map<Integer, ReduceFileGroups> reduceFileGroupsMap =
JavaUtils.newConcurrentHashMap();
- public ShuffleClientImpl(CelebornConf conf, UserIdentifier userIdentifier) {
+ public ShuffleClientImpl(String appUniqueId, CelebornConf conf, UserIdentifier userIdentifier) {
super();
+ this.appUniqueId = appUniqueId;
this.conf = conf;
this.userIdentifier = userIdentifier;
registerShuffleMaxRetries = conf.clientRegisterShuffleMaxRetry();
@@ -183,6 +186,8 @@ public class ShuffleClientImpl extends ShuffleClient {
partitionSplitPool =
ThreadUtils.newDaemonCachedThreadPool(
"celeborn-shuffle-split", pushSplitPartitionThreads, 60);
+
+ logger.info("Created ShuffleClientImpl, appUniqueId: {}", appUniqueId);
}
private boolean checkPushBlacklisted(
@@ -200,7 +205,6 @@ public class ShuffleClientImpl extends ShuffleClient {
}
private void submitRetryPushData(
- String applicationId,
int shuffleId,
int mapId,
int attemptId,
@@ -212,8 +216,7 @@ public class ShuffleClientImpl extends ShuffleClient {
StatusCode cause,
int remainReviveTimes) {
int partitionId = loc.getId();
- if (!revive(
- applicationId, shuffleId, mapId, attemptId, partitionId, loc.getEpoch(), loc, cause)) {
+ if (!revive(shuffleId, mapId, attemptId, partitionId, loc.getEpoch(), loc, cause)) {
wrappedCallback.onFailure(
new CelebornIOException(cause + " then revive but " + StatusCode.REVIVE_FAILED));
} else if (mapperEnded(shuffleId, mapId)) {
@@ -242,8 +245,7 @@ public class ShuffleClientImpl extends ShuffleClient {
TransportClient client =
dataClientFactory.createClient(newLoc.getHost(), newLoc.getPushPort(), partitionId);
NettyManagedBuffer newBuffer = new NettyManagedBuffer(Unpooled.wrappedBuffer(body));
- String shuffleKey = Utils.makeShuffleKey(applicationId, shuffleId);
-
+ String shuffleKey = Utils.makeShuffleKey(appUniqueId, shuffleId);
PushData newPushData =
new PushData(MASTER_MODE, shuffleKey, newLoc.getUniqueId(), newBuffer);
client.pushData(newPushData, pushDataTimeout, wrappedCallback);
@@ -272,7 +274,6 @@ public class ShuffleClientImpl extends ShuffleClient {
private void submitRetryPushMergedData(
PushState pushState,
- String applicationId,
int shuffleId,
int mapId,
int attemptId,
@@ -285,14 +286,7 @@ public class ShuffleClientImpl extends ShuffleClient {
for (DataBatches.DataBatch batch : batches) {
int partitionId = batch.loc.getId();
if (!revive(
- applicationId,
- shuffleId,
- mapId,
- attemptId,
- partitionId,
- batch.loc.getEpoch(),
- batch.loc,
- cause)) {
+ shuffleId, mapId, attemptId, partitionId, batch.loc.getEpoch(), batch.loc, cause)) {
if (remainReviveTimes > 0) {
reviveFailedBatchesMap.add(batch);
@@ -337,7 +331,6 @@ public class ShuffleClientImpl extends ShuffleClient {
DataBatches newDataBatches = entry.getValue();
doPushMergedData(
addressPair,
- applicationId,
shuffleId,
mapId,
attemptId,
@@ -352,7 +345,6 @@ public class ShuffleClientImpl extends ShuffleClient {
() ->
submitRetryPushMergedData(
pushState,
- applicationId,
shuffleId,
mapId,
attemptId,
@@ -374,22 +366,21 @@ public class ShuffleClientImpl extends ShuffleClient {
}
private ConcurrentHashMap<Integer, PartitionLocation> registerShuffle(
- String appId, int shuffleId, int numMappers, int numPartitions) {
+ int shuffleId, int numMappers, int numPartitions) {
return registerShuffleInternal(
shuffleId,
numMappers,
numPartitions,
() ->
driverRssMetaService.askSync(
- RegisterShuffle$.MODULE$.apply(appId, shuffleId, numMappers, numPartitions),
+ RegisterShuffle$.MODULE$.apply(shuffleId, numMappers, numPartitions),
conf.clientRpcRegisterShuffleRpcAskTimeout(),
ClassTag$.MODULE$.apply(PbRegisterShuffleResponse.class)));
}
@VisibleForTesting
public PartitionLocation registerMapPartitionTask(
- String appId, int shuffleId, int numMappers, int mapId, int attemptId, int partitionId)
- throws IOException {
+ int shuffleId, int numMappers, int mapId, int attemptId, int partitionId) throws IOException {
logger.info(
"Register MapPartition task for shuffle {} map {} attempt {} partition {} with {} mapper.",
shuffleId,
@@ -405,13 +396,12 @@ public class ShuffleClientImpl extends ShuffleClient {
() ->
driverRssMetaService.askSync(
RegisterMapPartitionTask$.MODULE$.apply(
- appId, shuffleId, numMappers, mapId, attemptId, partitionId),
+ shuffleId, numMappers, mapId, attemptId, partitionId),
conf.clientRpcRegisterShuffleRpcAskTimeout(),
ClassTag$.MODULE$.apply(PbRegisterShuffleResponse.class)));
if (partitionLocationMap == null) {
- String shuffleKey = Utils.makeShuffleKey(appId, shuffleId);
- throw new CelebornIOException("Register shuffle failed for shuffle " + shuffleKey);
+ throw new CelebornIOException("Register shuffle failed for shuffle " + shuffleId);
}
return partitionLocationMap.get(partitionId);
@@ -419,9 +409,9 @@ public class ShuffleClientImpl extends ShuffleClient {
@Override
public ConcurrentHashMap<Integer, PartitionLocation> getPartitionLocation(
- String applicationId, int shuffleId, int numMappers, int numPartitions) {
+ int shuffleId, int numMappers, int numPartitions) {
return reducePartitionMap.computeIfAbsent(
- shuffleId, (id) -> registerShuffle(applicationId, shuffleId, numMappers, numPartitions));
+ shuffleId, (id) -> registerShuffle(shuffleId, numMappers, numPartitions));
}
@Override
@@ -527,7 +517,6 @@ public class ShuffleClientImpl extends ShuffleClient {
}
private boolean revive(
- String applicationId,
int shuffleId,
int mapId,
int attemptId,
@@ -577,14 +566,7 @@ public class ShuffleClientImpl extends ShuffleClient {
PbChangeLocationResponse response =
driverRssMetaService.askSync(
Revive$.MODULE$.apply(
- applicationId,
- shuffleId,
- mapId,
- attemptId,
- partitionId,
- epoch,
- oldLocation,
- cause),
+ shuffleId, mapId, attemptId, partitionId, epoch, oldLocation, cause),
conf.clientRpcRequestPartitionLocationRpcAskTimeout(),
ClassTag$.MODULE$.apply(PbChangeLocationResponse.class));
// per partitionKey only serve single PartitionLocation in Client Cache.
@@ -624,7 +606,6 @@ public class ShuffleClientImpl extends ShuffleClient {
}
public int pushOrMergeData(
- String applicationId,
int shuffleId,
int mapId,
int attemptId,
@@ -638,7 +619,6 @@ public class ShuffleClientImpl extends ShuffleClient {
throws IOException {
// mapKey
final String mapKey = Utils.makeMapKey(shuffleId, mapId, attemptId);
- final String shuffleKey = Utils.makeShuffleKey(applicationId, shuffleId);
// return if shuffle stage already ended
if (mapperEnded(shuffleId, mapId)) {
logger.debug(
@@ -655,10 +635,10 @@ public class ShuffleClientImpl extends ShuffleClient {
}
// register shuffle if not registered
final ConcurrentHashMap<Integer, PartitionLocation> map =
- getPartitionLocation(applicationId, shuffleId, numMappers, numPartitions);
+ getPartitionLocation(shuffleId, numMappers, numPartitions);
if (map == null) {
- throw new CelebornIOException("Register shuffle failed for shuffle " + shuffleKey + ".");
+ throw new CelebornIOException("Register shuffle failed for shuffle " + shuffleId + ".");
}
// get location
@@ -666,7 +646,6 @@ public class ShuffleClientImpl extends ShuffleClient {
// register shuffle will return an empty location map, client need revive for a new location.
if (!map.containsKey(partitionId)) {
if (!revive(
- applicationId,
shuffleId,
mapId,
attemptId,
@@ -675,7 +654,7 @@ public class ShuffleClientImpl extends ShuffleClient {
null,
StatusCode.PUSH_DATA_FAIL_NON_CRITICAL_CAUSE)) {
throw new CelebornIOException(
- String.format("Revive for shuffle %s partition %d failed.", shuffleKey, partitionId));
+ String.format("Revive for shuffle %s partition %d failed.", shuffleId, partitionId));
}
}
@@ -697,7 +676,7 @@ public class ShuffleClientImpl extends ShuffleClient {
if (loc == null) {
throw new CelebornIOException(
String.format(
- "Partition location for shuffle %s partition %d is NULL!", shuffleKey, partitionId));
+ "Partition location for shuffle %s partition %d is NULL!", shuffleId, partitionId));
}
PushState pushState = getPushState(mapKey);
@@ -728,6 +707,7 @@ public class ShuffleClientImpl extends ShuffleClient {
// build PushData request
NettyManagedBuffer buffer = new NettyManagedBuffer(Unpooled.wrappedBuffer(body));
+ final String shuffleKey = Utils.makeShuffleKey(appUniqueId, shuffleId);
PushData pushData = new PushData(MASTER_MODE, shuffleKey, loc.getUniqueId(), buffer);
// build callback
@@ -778,7 +758,7 @@ public class ShuffleClientImpl extends ShuffleClient {
attemptId,
partitionId,
nextBatchId);
- splitPartition(shuffleId, partitionId, applicationId, loc);
+ splitPartition(shuffleId, partitionId, loc);
pushState.onSuccess(loc.hostAndPushPort());
callback.onSuccess(response);
} else if (reason == StatusCode.HARD_SPLIT.getValue()) {
@@ -793,7 +773,6 @@ public class ShuffleClientImpl extends ShuffleClient {
pushDataRetryPool.submit(
() ->
submitRetryPushData(
- applicationId,
shuffleId,
mapId,
attemptId,
@@ -874,7 +853,6 @@ public class ShuffleClientImpl extends ShuffleClient {
pushDataRetryPool.submit(
() ->
submitRetryPushData(
- applicationId,
shuffleId,
mapId,
attemptId,
@@ -937,7 +915,6 @@ public class ShuffleClientImpl extends ShuffleClient {
DataBatches dataBatches = pushState.takeDataBatches(addressPair);
doPushMergedData(
addressPair,
- applicationId,
shuffleId,
mapId,
attemptId,
@@ -950,8 +927,7 @@ public class ShuffleClientImpl extends ShuffleClient {
return body.length;
}
- private void splitPartition(
- int shuffleId, int partitionId, String applicationId, PartitionLocation loc) {
+ private void splitPartition(int shuffleId, int partitionId, PartitionLocation loc) {
Set<Integer> splittingSet =
splitting.computeIfAbsent(shuffleId, integer -> ConcurrentHashMap.newKeySet());
//noinspection SynchronizationOnLocalVariableOrMethodParameter
@@ -970,7 +946,7 @@ public class ShuffleClientImpl extends ShuffleClient {
ShuffleClientHelper.sendShuffleSplitAsync(
driverRssMetaService,
conf,
- PartitionSplit$.MODULE$.apply(applicationId, shuffleId, partitionId, loc.getEpoch(), loc),
+ PartitionSplit$.MODULE$.apply(shuffleId, partitionId, loc.getEpoch(), loc),
partitionSplitPool,
splittingSet,
partitionId,
@@ -980,7 +956,6 @@ public class ShuffleClientImpl extends ShuffleClient {
@Override
public int pushData(
- String applicationId,
int shuffleId,
int mapId,
int attemptId,
@@ -992,7 +967,6 @@ public class ShuffleClientImpl extends ShuffleClient {
int numPartitions)
throws IOException {
return pushOrMergeData(
- applicationId,
shuffleId,
mapId,
attemptId,
@@ -1016,7 +990,6 @@ public class ShuffleClientImpl extends ShuffleClient {
@Override
public int mergeData(
- String applicationId,
int shuffleId,
int mapId,
int attemptId,
@@ -1028,7 +1001,6 @@ public class ShuffleClientImpl extends ShuffleClient {
int numPartitions)
throws IOException {
return pushOrMergeData(
- applicationId,
shuffleId,
mapId,
attemptId,
@@ -1041,8 +1013,7 @@ public class ShuffleClientImpl extends ShuffleClient {
false);
}
- public void pushMergedData(String applicationId, int shuffleId, int mapId, int attemptId)
- throws IOException {
+ public void pushMergedData(int shuffleId, int mapId, int attemptId) throws IOException {
final String mapKey = Utils.makeMapKey(shuffleId, mapId, attemptId);
PushState pushState = pushStates.get(mapKey);
if (pushState == null) {
@@ -1059,20 +1030,12 @@ public class ShuffleClientImpl extends ShuffleClient {
batchesArr.remove(entry);
}
doPushMergedData(
- entry.getKey(),
- applicationId,
- shuffleId,
- mapId,
- attemptId,
- batches,
- pushState,
- maxReviveTimes);
+ entry.getKey(), shuffleId, mapId, attemptId, batches, pushState, maxReviveTimes);
}
}
private void doPushMergedData(
String addressPair,
- String applicationId,
int shuffleId,
int mapId,
int attemptId,
@@ -1104,7 +1067,7 @@ public class ShuffleClientImpl extends ShuffleClient {
byteBuf.addComponent(true, Unpooled.wrappedBuffer(batch.body));
}
NettyManagedBuffer buffer = new NettyManagedBuffer(byteBuf);
- String shuffleKey = Utils.makeShuffleKey(applicationId, shuffleId);
+ String shuffleKey = Utils.makeShuffleKey(appUniqueId, shuffleId);
PushMergedData mergedData =
new PushMergedData(MASTER_MODE, shuffleKey, partitionUniqueIds, offsets, buffer);
@@ -1179,7 +1142,6 @@ public class ShuffleClientImpl extends ShuffleClient {
() ->
submitRetryPushMergedData(
pushState,
- applicationId,
shuffleId,
mapId,
attemptId,
@@ -1260,7 +1222,6 @@ public class ShuffleClientImpl extends ShuffleClient {
() ->
submitRetryPushMergedData(
pushState,
- applicationId,
shuffleId,
mapId,
attemptId,
@@ -1314,31 +1275,19 @@ public class ShuffleClientImpl extends ShuffleClient {
}
@Override
- public void mapperEnd(
- String applicationId, int shuffleId, int mapId, int attemptId, int numMappers)
+ public void mapperEnd(int shuffleId, int mapId, int attemptId, int numMappers)
throws IOException {
- mapEndInternal(applicationId, shuffleId, mapId, attemptId, numMappers, -1);
+ mapEndInternal(shuffleId, mapId, attemptId, numMappers, -1);
}
@Override
public void mapPartitionMapperEnd(
- String applicationId,
- int shuffleId,
- int mapId,
- int attemptId,
- int numMappers,
- int partitionId)
- throws IOException {
- mapEndInternal(applicationId, shuffleId, mapId, attemptId, numMappers, partitionId);
+ int shuffleId, int mapId, int attemptId, int numMappers, int partitionId) throws IOException {
+ mapEndInternal(shuffleId, mapId, attemptId, numMappers, partitionId);
}
private void mapEndInternal(
- String applicationId,
- int shuffleId,
- int mapId,
- int attemptId,
- int numMappers,
- Integer partitionId)
+ int shuffleId, int mapId, int attemptId, int numMappers, Integer partitionId)
throws IOException {
final String mapKey = Utils.makeMapKey(shuffleId, mapId, attemptId);
PushState pushState = getPushState(mapKey);
@@ -1348,7 +1297,7 @@ public class ShuffleClientImpl extends ShuffleClient {
MapperEndResponse response =
driverRssMetaService.askSync(
- new MapperEnd(applicationId, shuffleId, mapId, attemptId, numMappers, partitionId),
+ new MapperEnd(shuffleId, mapId, attemptId, numMappers, partitionId),
ClassTag$.MODULE$.apply(MapperEndResponse.class));
if (response.status() != StatusCode.SUCCESS) {
throw new CelebornIOException("MapperEnd failed! StatusCode: " + response.status());
@@ -1359,7 +1308,7 @@ public class ShuffleClientImpl extends ShuffleClient {
}
@Override
- public void cleanup(String applicationId, int shuffleId, int mapId, int attemptId) {
+ public void cleanup(int shuffleId, int mapId, int attemptId) {
final String mapKey = Utils.makeMapKey(shuffleId, mapId, attemptId);
PushState pushState = pushStates.remove(mapKey);
if (pushState != null) {
@@ -1369,12 +1318,12 @@ public class ShuffleClientImpl extends ShuffleClient {
}
@Override
- public boolean unregisterShuffle(String applicationId, int shuffleId, boolean isDriver) {
+ public boolean unregisterShuffle(int shuffleId, boolean isDriver) {
if (isDriver) {
try {
driverRssMetaService.send(
UnregisterShuffle$.MODULE$.apply(
- applicationId, shuffleId, RssHARetryClient.genRequestId()));
+ appUniqueId, shuffleId, RssHARetryClient.genRequestId()));
} catch (Exception e) {
// If some exceptions need to be ignored, they shouldn't be logged as error-level,
// otherwise it will mislead users.
@@ -1394,14 +1343,12 @@ public class ShuffleClientImpl extends ShuffleClient {
}
@Override
- public RssInputStream readPartition(
- String applicationId, int shuffleId, int partitionId, int attemptNumber) throws IOException {
- return readPartition(
- applicationId, shuffleId, partitionId, attemptNumber, 0, Integer.MAX_VALUE);
+ public RssInputStream readPartition(int shuffleId, int partitionId, int attemptNumber)
+ throws IOException {
+ return readPartition(shuffleId, partitionId, attemptNumber, 0, Integer.MAX_VALUE);
}
- protected ReduceFileGroups loadFileGroupInternal(
- String applicationId, String shuffleKey, int shuffleId) {
+ protected ReduceFileGroups loadFileGroupInternal(int shuffleId) {
{
long getReducerFileGroupStartTime = System.nanoTime();
try {
@@ -1410,7 +1357,7 @@ public class ShuffleClientImpl extends ShuffleClient {
return null;
}
- GetReducerFileGroup getReducerFileGroup = new GetReducerFileGroup(applicationId, shuffleId);
+ GetReducerFileGroup getReducerFileGroup = new GetReducerFileGroup(shuffleId);
GetReducerFileGroupResponse response =
driverRssMetaService.askSync(
@@ -1431,31 +1378,27 @@ public class ShuffleClientImpl extends ShuffleClient {
"Request {} return {} for {}.",
getReducerFileGroup,
StatusCode.STAGE_END_TIME_OUT,
- shuffleKey);
+ shuffleId);
} else if (response.status() == StatusCode.SHUFFLE_DATA_LOST) {
logger.warn(
"Request {} return {} for {}.",
getReducerFileGroup,
StatusCode.SHUFFLE_DATA_LOST,
- shuffleKey);
+ shuffleId);
}
} catch (Exception e) {
- logger.error("Exception raised while call GetReducerFileGroup for {}.", shuffleKey, e);
+ logger.error("Exception raised while call GetReducerFileGroup for {}.", shuffleId, e);
}
return null;
}
}
- protected ReduceFileGroups updateFileGroup(
- String applicationId, String shuffleKey, int shuffleId, int partitionId) throws IOException {
- return reduceFileGroupsMap.computeIfAbsent(
- shuffleId, (id) -> loadFileGroupInternal(applicationId, shuffleKey, shuffleId));
+ protected ReduceFileGroups updateFileGroup(int shuffleId, int partitionId) throws IOException {
+ return reduceFileGroupsMap.computeIfAbsent(shuffleId, (id) -> loadFileGroupInternal(shuffleId));
}
- protected ReduceFileGroups loadFileGroup(
- String applicationId, String shuffleKey, int shuffleId, int partitionId) throws IOException {
- ReduceFileGroups reduceFileGroups =
- updateFileGroup(applicationId, shuffleKey, shuffleId, partitionId);
+ protected ReduceFileGroups loadFileGroup(int shuffleId, int partitionId) throws IOException {
+ ReduceFileGroups reduceFileGroups = updateFileGroup(shuffleId, partitionId);
if (reduceFileGroups == null) {
String msg =
"Shuffle data lost for shuffle " + shuffleId + " partitionId " + partitionId + "!";
@@ -1467,21 +1410,16 @@ public class ShuffleClientImpl extends ShuffleClient {
@Override
public RssInputStream readPartition(
- String applicationId,
- int shuffleId,
- int partitionId,
- int attemptNumber,
- int startMapIndex,
- int endMapIndex)
+ int shuffleId, int partitionId, int attemptNumber, int startMapIndex, int endMapIndex)
throws IOException {
- String shuffleKey = Utils.makeShuffleKey(applicationId, shuffleId);
- ReduceFileGroups fileGroups = loadFileGroup(applicationId, shuffleKey, shuffleId, partitionId);
+ ReduceFileGroups fileGroups = loadFileGroup(shuffleId, partitionId);
if (fileGroups.partitionGroups.size() == 0
|| !fileGroups.partitionGroups.containsKey(partitionId)) {
logger.warn("Shuffle data is empty for shuffle {} partition {}.", shuffleId, partitionId);
return RssInputStream.empty();
} else {
+ String shuffleKey = Utils.makeShuffleKey(appUniqueId, shuffleId);
return RssInputStream.create(
conf,
dataClientFactory,
diff --git a/client/src/main/java/org/apache/celeborn/client/write/DataPushQueue.java b/client/src/main/java/org/apache/celeborn/client/write/DataPushQueue.java
index d6070d161..750a4e6c3 100644
--- a/client/src/main/java/org/apache/celeborn/client/write/DataPushQueue.java
+++ b/client/src/main/java/org/apache/celeborn/client/write/DataPushQueue.java
@@ -48,7 +48,6 @@ public class DataPushQueue {
private final DataPusher dataPusher;
private final int maxInFlight;
- private final String appId;
private final int shuffleId;
private final int numMappers;
private final int numPartitions;
@@ -60,13 +59,11 @@ public class DataPushQueue {
CelebornConf conf,
DataPusher dataPusher,
ShuffleClient client,
- String appId,
int shuffleId,
int mapId,
int attemptId,
int numMappers,
int numPartitions) {
- this.appId = appId;
this.shuffleId = shuffleId;
this.numMappers = numMappers;
this.numPartitions = numPartitions;
@@ -99,7 +96,7 @@ public class DataPushQueue {
PushTask task = iterator.next();
int partitionId = task.getPartitionId();
Map<Integer, PartitionLocation> partitionLocationMap =
- client.getPartitionLocation(appId, shuffleId, numMappers, numPartitions);
+ client.getPartitionLocation(shuffleId, numMappers, numPartitions);
if (partitionLocationMap != null) {
PartitionLocation loc = partitionLocationMap.get(partitionId);
// According to CELEBORN-560, call rerun task and speculative task after LifecycleManager
diff --git a/client/src/main/java/org/apache/celeborn/client/write/DataPusher.java b/client/src/main/java/org/apache/celeborn/client/write/DataPusher.java
index b35f58bb2..bc26dab69 100644
--- a/client/src/main/java/org/apache/celeborn/client/write/DataPusher.java
+++ b/client/src/main/java/org/apache/celeborn/client/write/DataPusher.java
@@ -48,7 +48,6 @@ public class DataPusher {
private final AtomicReference<IOException> exceptionRef = new AtomicReference<>();
- private final String appId;
private final int shuffleId;
private final int mapId;
private final int attemptId;
@@ -62,7 +61,6 @@ public class DataPusher {
private Thread pushThread;
public DataPusher(
- String appId,
int shuffleId,
int mapId,
int attemptId,
@@ -80,13 +78,12 @@ public class DataPusher {
idleQueue = new LinkedBlockingQueue<>(pushQueueCapacity);
dataPushQueue =
new DataPushQueue(
- conf, this, client, appId, shuffleId, mapId, attemptId, numMappers, numPartitions);
+ conf, this, client, shuffleId, mapId, attemptId, numMappers, numPartitions);
for (int i = 0; i < pushQueueCapacity; i++) {
idleQueue.put(new PushTask(pushBufferMaxSize));
}
- this.appId = appId;
this.shuffleId = shuffleId;
this.mapId = mapId;
this.attemptId = attemptId;
@@ -195,7 +192,6 @@ public class DataPusher {
private void pushData(PushTask task) throws IOException {
int bytesWritten =
client.pushData(
- appId,
shuffleId,
mapId,
attemptId,
diff --git a/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala b/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala
index a5050ef3c..beb9cb70c 100644
--- a/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala
@@ -34,7 +34,6 @@ import org.apache.celeborn.common.util.{JavaUtils, ThreadUtils, Utils}
case class ChangePartitionRequest(
context: RequestLocationCallContext,
- applicationId: String,
shuffleId: Int,
partitionId: Int,
epoch: Int,
@@ -89,7 +88,6 @@ class ChangePartitionManager(
}.toArray
if (distinctPartitions.nonEmpty) {
handleRequestPartitions(
- distinctPartitions.head.applicationId,
shuffleId,
distinctPartitions)
}
@@ -130,7 +128,6 @@ class ChangePartitionManager(
def handleRequestPartitionLocation(
context: RequestLocationCallContext,
- applicationId: String,
shuffleId: Int,
partitionId: Int,
oldEpoch: Int,
@@ -139,7 +136,6 @@ class ChangePartitionManager(
val changePartition = ChangePartitionRequest(
context,
- applicationId,
shuffleId,
partitionId,
oldEpoch,
@@ -178,7 +174,7 @@ class ChangePartitionManager(
}
}
if (!batchHandleChangePartitionEnabled) {
- handleRequestPartitions(applicationId, shuffleId, Array(changePartition))
+ handleRequestPartitions(shuffleId, Array(changePartition))
}
}
@@ -197,7 +193,6 @@ class ChangePartitionManager(
}
def handleRequestPartitions(
- applicationId: String,
shuffleId: Int,
changePartitions: Array[ChangePartitionRequest]): Unit = {
val requestsMap = changePartitionRequests.get(shuffleId)
@@ -205,7 +200,7 @@ class ChangePartitionManager(
val changes = changePartitions.map { change =>
s"${change.shuffleId}-${change.partitionId}-${change.epoch}"
}.mkString("[", ",", "]")
- logWarning(s"Batch handle change partition for $applicationId of $changes")
+ logWarning(s"Batch handle change partition for $changes")
// Blacklist all failed workers
if (changePartitions.exists(_.causes.isDefined)) {
@@ -275,7 +270,6 @@ class ChangePartitionManager(
reallocateChangePartitionRequestSlotsFromCandidates(changePartitions.toList, candidates)
if (!lifecycleManager.reserveSlotsWithRetry(
- applicationId,
shuffleId,
new util.HashSet(candidates.toSet.asJava),
newlyAllocatedLocations)) {
@@ -303,7 +297,7 @@ class ChangePartitionManager(
s"(partition ${partition.getId} epoch from ${partition.getEpoch - 1} to ${partition.getEpoch})"
}.mkString("[", ", ", "]")
logDebug(s"[Update partition] success for " +
- s"shuffle ${Utils.makeShuffleKey(applicationId, shuffleId)}, succeed partitions: " +
+ s"shuffle $shuffleId, succeed partitions: " +
s"$changes.")
}
locations
diff --git a/client/src/main/scala/org/apache/celeborn/client/CommitManager.scala b/client/src/main/scala/org/apache/celeborn/client/CommitManager.scala
index 228eb6e17..d80910d9c 100644
--- a/client/src/main/scala/org/apache/celeborn/client/CommitManager.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/CommitManager.scala
@@ -69,7 +69,7 @@ object CommitManager {
type CommittedPartitionInfo = ConcurrentHashMap[Int, ShuffleCommittedInfo]
}
-class CommitManager(appId: String, val conf: CelebornConf, lifecycleManager: LifecycleManager)
+class CommitManager(appUniqueId: String, val conf: CelebornConf, lifecycleManager: LifecycleManager)
extends Logging {
// shuffle id -> ShuffleCommittedInfo
@@ -142,7 +142,7 @@ class CommitManager(appId: String, val conf: CelebornConf, lifecycleManager: Lif
.asJava
commitHandler.commitFiles(
- appId,
+ appUniqueId,
shuffleId,
shuffleCommittedInfo,
workerInfo,
@@ -272,13 +272,13 @@ class CommitManager(appId: String, val conf: CelebornConf, lifecycleManager: Lif
(partitionType: PartitionType) => {
partitionType match {
case PartitionType.REDUCE => new ReducePartitionCommitHandler(
- appId,
+ appUniqueId,
conf,
lifecycleManager.shuffleAllocatedWorkers,
committedPartitionInfo,
lifecycleManager.workerStatusTracker)
case PartitionType.MAP => new MapPartitionCommitHandler(
- appId,
+ appUniqueId,
conf,
lifecycleManager.shuffleAllocatedWorkers,
committedPartitionInfo,
diff --git a/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala b/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
index b77efea88..f6e4586cc 100644
--- a/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
@@ -52,7 +52,8 @@ object LifecycleManager {
type ShuffleFailedWorkers = ConcurrentHashMap[WorkerInfo, (StatusCode, Long)]
}
-class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoint with Logging {
+class LifecycleManager(val appUniqueId: String, val conf: CelebornConf) extends RpcEndpoint
+ with Logging {
private val lifecycleHost = Utils.localHostName
@@ -118,17 +119,17 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
logInfo(s"Starting LifecycleManager on ${rpcEnv.address}")
private val rssHARetryClient = new RssHARetryClient(rpcEnv, conf)
- val commitManager = new CommitManager(appId, conf, this)
+ val commitManager = new CommitManager(appUniqueId, conf, this)
val workerStatusTracker = new WorkerStatusTracker(conf, this)
private val heartbeater =
new ApplicationHeartbeater(
- appId,
+ appUniqueId,
conf,
rssHARetryClient,
() => commitManager.commitMetrics(),
workerStatusTracker)
private val changePartitionManager = new ChangePartitionManager(conf, this)
- private val releasePartitionManager = new ReleasePartitionManager(appId, conf, this)
+ private val releasePartitionManager = new ReleasePartitionManager(conf, this)
// Since method `onStart` is executed when `rpcEnv.setupEndpoint` is executed, and
// `rssHARetryClient` is initialized after `rpcEnv` is initialized, if method `onStart` contains
@@ -193,52 +194,46 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
override def receive: PartialFunction[Any, Unit] = {
case RemoveExpiredShuffle =>
removeExpiredShuffle()
- case StageEnd(applicationId, shuffleId) =>
- logInfo(s"Received StageEnd request, ${Utils.makeShuffleKey(applicationId, shuffleId)}.")
- handleStageEnd(applicationId, shuffleId)
+ case StageEnd(shuffleId) =>
+ logInfo(s"Received StageEnd request, shuffleId $shuffleId.")
+ handleStageEnd(shuffleId)
case pb: PbUnregisterShuffle =>
- val applicationId = pb.getAppId
val shuffleId = pb.getShuffleId
logDebug(s"Received UnregisterShuffle request," +
- s"${Utils.makeShuffleKey(applicationId, shuffleId)}.")
- handleUnregisterShuffle(applicationId, shuffleId)
+ s"shuffleId $shuffleId.")
+ handleUnregisterShuffle(shuffleId)
}
override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
case pb: PbRegisterShuffle =>
- val applicationId = pb.getApplicationId
val shuffleId = pb.getShuffleId
val numMappers = pb.getNumMapppers
val numPartitions = pb.getNumPartitions
logDebug(s"Received RegisterShuffle request, " +
- s"$applicationId, $shuffleId, $numMappers, $numPartitions.")
+ s"$shuffleId, $numMappers, $numPartitions.")
offerAndReserveSlots(
RegisterCallContext(context),
- applicationId,
shuffleId,
numMappers,
numPartitions)
case pb: PbRegisterMapPartitionTask =>
- val applicationId = pb.getApplicationId
val shuffleId = pb.getShuffleId
val numMappers = pb.getNumMappers
val mapId = pb.getMapId
val attemptId = pb.getAttemptId
val partitionId = pb.getPartitionId
logDebug(s"Received Register map partition task request, " +
- s"$applicationId, $shuffleId, $numMappers, $mapId, $attemptId, $partitionId.")
+ s"$shuffleId, $numMappers, $mapId, $attemptId, $partitionId.")
shufflePartitionType.putIfAbsent(shuffleId, PartitionType.MAP)
offerAndReserveSlots(
RegisterCallContext(context, partitionId),
- applicationId,
shuffleId,
numMappers,
numMappers,
partitionId)
case pb: PbRevive =>
- val applicationId = pb.getApplicationId
val shuffleId = pb.getShuffleId
val mapId = pb.getMapId
val attemptId = pb.getAttemptId
@@ -252,11 +247,10 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
}
val cause = Utils.toStatusCode(pb.getStatus)
logTrace(s"Received Revive request, " +
- s"$applicationId, $shuffleId, $mapId, $attemptId ,$partitionId," +
+ s"$shuffleId, $mapId, $attemptId ,$partitionId," +
s" $epoch, $oldPartition, $cause.")
handleRevive(
context,
- applicationId,
shuffleId,
mapId,
attemptId,
@@ -266,32 +260,29 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
cause)
case pb: PbPartitionSplit =>
- val applicationId = pb.getApplicationId
val shuffleId = pb.getShuffleId
val partitionId = pb.getPartitionId
val epoch = pb.getEpoch
val oldPartition = PbSerDeUtils.fromPbPartitionLocation(pb.getOldPartition)
logTrace(s"Received split request, " +
- s"$applicationId, $shuffleId, $partitionId, $epoch, $oldPartition")
+ s"$shuffleId, $partitionId, $epoch, $oldPartition")
changePartitionManager.handleRequestPartitionLocation(
ChangeLocationCallContext(context),
- applicationId,
shuffleId,
partitionId,
epoch,
oldPartition)
- case MapperEnd(applicationId, shuffleId, mapId, attemptId, numMappers, partitionId) =>
+ case MapperEnd(shuffleId, mapId, attemptId, numMappers, partitionId) =>
logTrace(s"Received MapperEnd TaskEnd request, " +
- s"${Utils.makeMapKey(applicationId, shuffleId, mapId, attemptId)}")
+ s"${Utils.makeMapKey(shuffleId, mapId, attemptId)}")
val partitionType = getPartitionType(shuffleId)
partitionType match {
case PartitionType.REDUCE =>
- handleMapperEnd(context, applicationId, shuffleId, mapId, attemptId, numMappers)
+ handleMapperEnd(context, shuffleId, mapId, attemptId, numMappers)
case PartitionType.MAP =>
handleMapPartitionEnd(
context,
- applicationId,
shuffleId,
mapId,
attemptId,
@@ -301,15 +292,14 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
throw new UnsupportedOperationException(s"Not support $partitionType yet")
}
- case GetReducerFileGroup(applicationId: String, shuffleId: Int) =>
+ case GetReducerFileGroup(shuffleId: Int) =>
logDebug(s"Received GetShuffleFileGroup request," +
- s"${Utils.makeShuffleKey(applicationId, shuffleId)}.")
+ s"shuffleId $shuffleId.")
handleGetReducerFileGroup(context, shuffleId)
}
private def offerAndReserveSlots(
context: RegisterCallContext,
- applicationId: String,
shuffleId: Int,
numMappers: Int,
numPartitions: Int,
@@ -335,7 +325,6 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
.toArray
partitionType match {
case PartitionType.MAP => processMapTaskReply(
- applicationId,
shuffleId,
context.context,
partitionId,
@@ -357,7 +346,6 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
}
def processMapTaskReply(
- applicationId: String,
shuffleId: Int,
context: RpcCallContext,
partitionId: Int,
@@ -369,7 +357,6 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
// request new resource for this task
changePartitionManager.handleRequestPartitionLocation(
ApplyNewLocationCallContext(context),
- applicationId,
shuffleId,
partitionId,
-1,
@@ -391,7 +378,6 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
_.getId == context.partitionId).map(r =>
PbSerDeUtils.fromPbPartitionLocation(r)).toArray
processMapTaskReply(
- applicationId,
shuffleId,
context.context,
context.partitionId,
@@ -413,7 +399,7 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
// First, request to get allocated slots from Master
val ids = new util.ArrayList[Integer](numPartitions)
(0 until numPartitions).foreach(idx => ids.add(Integer.valueOf(idx)))
- val res = requestMasterRequestSlotsWithRetry(applicationId, shuffleId, ids)
+ val res = requestMasterRequestSlotsWithRetry(shuffleId, ids)
res.status match {
case StatusCode.REQUEST_FAILED =>
@@ -462,7 +448,6 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
// and prepare the pushing data env.
val reserveSlotsSuccess =
reserveSlotsWithRetry(
- applicationId,
shuffleId,
candidatesWorkers,
slots,
@@ -474,7 +459,7 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
reply(RegisterShuffleResponse(StatusCode.RESERVE_SLOTS_FAILED, Array.empty))
// tell Master to release slots
requestMasterReleaseSlots(
- ReleaseSlots(applicationId, shuffleId, List.empty.asJava, List.empty.asJava))
+ ReleaseSlots(appUniqueId, shuffleId, List.empty.asJava, List.empty.asJava))
} else {
logInfo(s"ReserveSlots for $shuffleId success!")
logDebug(s"Allocated Slots: $slots")
@@ -501,7 +486,6 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
private def handleRevive(
context: RpcCallContext,
- applicationId: String,
shuffleId: Int,
mapId: Int,
attemptId: Int,
@@ -538,12 +522,11 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
return
}
- logWarning(s"Do Revive for shuffle ${Utils.makeShuffleKey(applicationId, shuffleId)}, " +
+ logWarning(s"Do Revive for shuffle shuffleId $shuffleId, " +
s"oldPartition: $oldPartition, cause: $cause")
changePartitionManager.handleRequestPartitionLocation(
ChangeLocationCallContext(context),
- applicationId,
shuffleId,
partitionId,
oldEpoch,
@@ -553,7 +536,6 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
private def handleMapperEnd(
context: RpcCallContext,
- applicationId: String,
shuffleId: Int,
mapId: Int,
attemptId: Int,
@@ -564,8 +546,8 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
if (mapperAttemptFinishedSuccess && allMapperFinished) {
// last mapper finished. call mapper end
logInfo(s"Last MapperEnd, call StageEnd with shuffleKey:" +
- s"${Utils.makeShuffleKey(applicationId, shuffleId)}.")
- self.send(StageEnd(applicationId, shuffleId))
+ s"shuffleId $shuffleId.")
+ self.send(StageEnd(shuffleId))
}
// reply success
@@ -578,7 +560,7 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
commitManager.handleGetReducerFileGroup(context, shuffleId)
}
- private def handleStageEnd(applicationId: String, shuffleId: Int): Unit = {
+ private def handleStageEnd(shuffleId: Int): Unit = {
// check whether shuffle has registered
if (!registeredShuffle.contains(shuffleId)) {
logInfo(s"[handleStageEnd]" +
@@ -596,13 +578,12 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
partitionLocationInfo.removeAllSlavePartitions()
}
requestMasterReleaseSlots(
- ReleaseSlots(applicationId, shuffleId, List.empty.asJava, List.empty.asJava))
+ ReleaseSlots(appUniqueId, shuffleId, List.empty.asJava, List.empty.asJava))
}
}
private def handleMapPartitionEnd(
context: RpcCallContext,
- applicationId: String,
shuffleId: Int,
mapId: Int,
attemptId: Int,
@@ -610,7 +591,7 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
numMappers: Int): Unit = {
def reply(result: Boolean): Unit = {
val message =
- s"to handle MapPartitionEnd for ${Utils.makeMapKey(applicationId, shuffleId, mapId, attemptId)}, " +
+ s"to handle MapPartitionEnd for ${Utils.makeMapKey(shuffleId, mapId, attemptId)}, " +
s"$partitionId.";
result match {
case true => // if already committed by another try
@@ -632,14 +613,13 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
}
def handleUnregisterShuffle(
- appId: String,
shuffleId: Int): Unit = {
if (getPartitionType(shuffleId) == PartitionType.REDUCE) {
// if StageEnd has not been handled, trigger StageEnd
if (!commitManager.isStageEnd(shuffleId)) {
logInfo(s"Call StageEnd before Unregister Shuffle $shuffleId.")
// try call stage end
- handleStageEnd(appId, shuffleId)
+ handleStageEnd(shuffleId)
// wait stage end
val (isTimeOut, cost) = commitManager.waitStageEnd(shuffleId)
if (isTimeOut) {
@@ -654,7 +634,7 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
logWarning(s"Partition exists for shuffle $shuffleId, " +
"maybe caused by task rerun or speculative.")
requestMasterReleaseSlots(
- ReleaseSlots(appId, shuffleId, List.empty.asJava, List.empty.asJava))
+ ReleaseSlots(appUniqueId, shuffleId, List.empty.asJava, List.empty.asJava))
}
// add shuffleKey to delay shuffle removal set
@@ -671,13 +651,11 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
* After getting WorkerResource, LifecycleManger needs to ask each Worker to
* reserve corresponding slot and prepare push data env in Worker side.
*
- * @param applicationId Application ID
* @param shuffleId Application shuffle id
* @param slots WorkerResource to reserve slots
* @return List of reserving slot failed workers
*/
private def reserveSlots(
- applicationId: String,
shuffleId: Int,
slots: WorkerResource): util.List[WorkerInfo] = {
val reserveSlotFailedWorkers = new ShuffleFailedWorkers()
@@ -690,7 +668,7 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
val res = requestWorkerReserveSlots(
workerInfo.endpoint,
ReserveSlots(
- applicationId,
+ appUniqueId,
shuffleId,
masterLocations,
slaveLocations,
@@ -702,18 +680,18 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
conf.pushDataTimeoutMs))
if (res.status.equals(StatusCode.SUCCESS)) {
logDebug(s"Successfully allocated " +
- s"partitions buffer for ${Utils.makeShuffleKey(applicationId, shuffleId)}" +
+ s"partitions buffer for shuffleId $shuffleId" +
s" from worker ${workerInfo.readableAddress()}.")
} else {
failureInfos.add(s"[reserveSlots] Failed to" +
- s" reserve buffers for ${Utils.makeShuffleKey(applicationId, shuffleId)}" +
+ s" reserve buffers for shuffleId $shuffleId" +
s" from worker ${workerInfo.readableAddress()}. Reason: ${res.reason}")
reserveSlotFailedWorkers.put(workerInfo, (res.status, System.currentTimeMillis()))
}
}
if (failureInfos.asScala.nonEmpty) {
logError(s"Aggregated error of reserveSlots for " +
- s"${Utils.makeShuffleKey(applicationId, shuffleId)} " +
+ s"shuffleId $shuffleId " +
s"failure:${failureInfos.asScala.foldLeft("")((x, y) => s"$x \n $y")}")
}
workerStatusTracker.recordWorkerFailure(reserveSlotFailedWorkers)
@@ -728,13 +706,11 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
* 2. Request the Worker to destroy the slot's FileWriter.
* 3. Request the Master to release the worker slots status.
*
- * @param applicationId application id
* @param shuffleId shuffle id
* @param slots allocated WorkerResource
* @param failedPartitionLocations reserve slot failed partition location
*/
private def releasePeerPartitionLocation(
- applicationId: String,
shuffleId: Int,
slots: WorkerResource,
failedPartitionLocations: mutable.HashMap[Int, PartitionLocation]): Unit = {
@@ -778,9 +754,9 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
}
}
if (!destroyResource.isEmpty) {
- destroySlotsWithRetry(applicationId, shuffleId, destroyResource)
+ destroySlotsWithRetry(shuffleId, destroyResource)
logInfo(s"Destroyed peer partitions for reserve buffer failed workers " +
- s"${Utils.makeShuffleKey(applicationId, shuffleId)}, $destroyResource")
+ s"shuffleId $shuffleId, $destroyResource")
val workerIds = new util.ArrayList[String]()
val workerSlotsPerDisk = new util.ArrayList[util.Map[String, Integer]]()
@@ -789,11 +765,11 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
workerIds.add(workerInfo.toUniqueId())
workerSlotsPerDisk.add(slotsPerDisk)
}
- val msg = ReleaseSlots(applicationId, shuffleId, workerIds, workerSlotsPerDisk)
+ val msg = ReleaseSlots(appUniqueId, shuffleId, workerIds, workerSlotsPerDisk)
requestMasterReleaseSlots(msg)
logInfo(s"Released slots for reserve buffer failed workers " +
s"${workerIds.asScala.mkString(",")}" + s"${slots.asScala.mkString(",")}" +
- s"${Utils.makeShuffleKey(applicationId, shuffleId)}, ")
+ s"shuffleId $shuffleId")
}
}
@@ -834,14 +810,12 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
/**
* Reserve buffers with retry, retry on another node will cause slots to be inconsistent.
*
- * @param applicationId application id
* @param shuffleId shuffle id
* @param candidates working worker list
* @param slots the total allocated worker resources that need to be applied for the slot
* @return If reserve all slots success
*/
def reserveSlotsWithRetry(
- applicationId: String,
shuffleId: Int,
candidates: util.HashSet[WorkerInfo],
slots: WorkerResource,
@@ -858,7 +832,7 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
}
// reserve buffers
logInfo(s"Try reserve slots for $shuffleId for $retryTimes times.")
- val reserveFailedWorkers = reserveSlots(applicationId, shuffleId, requestSlots)
+ val reserveFailedWorkers = reserveSlots(shuffleId, requestSlots)
if (reserveFailedWorkers.isEmpty) {
success = true
} else {
@@ -870,7 +844,7 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
// When enable replicate, if one of the partition location reserve slots failed, we also
// need to release another corresponding partition location and remove it from slots.
if (pushReplicateEnabled && failedPartitionLocations.nonEmpty && !slots.isEmpty) {
- releasePeerPartitionLocation(applicationId, shuffleId, slots, failedPartitionLocations)
+ releasePeerPartitionLocation(shuffleId, slots, failedPartitionLocations)
}
if (retryTimes < reserveSlotsMaxRetries) {
// get retryCandidates resource and retry reserve buffer
@@ -911,9 +885,9 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
// [[releasePeerPartitionLocation]]. Now in the slots are all the successful partition
// locations.
logWarning(s"Reserve buffers for $shuffleId still fail after retrying, clear buffers.")
- destroySlotsWithRetry(applicationId, shuffleId, slots)
+ destroySlotsWithRetry(shuffleId, slots)
} else {
- logInfo(s"Reserve buffer success for ${Utils.makeShuffleKey(applicationId, shuffleId)}")
+ logInfo(s"Reserve buffer success for shuffleId $shuffleId")
}
success
}
@@ -997,16 +971,14 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
* For the slots that need to be destroyed, LifecycleManager will ask the corresponding worker
* to destroy related FileWriter.
*
- * @param applicationId application id
* @param shuffleId shuffle id
* @param slotsToDestroy worker resource to be destroyed
* @return destroy failed master and slave location unique id
*/
def destroySlotsWithRetry(
- applicationId: String,
shuffleId: Int,
slotsToDestroy: WorkerResource): Unit = {
- val shuffleKey = Utils.makeShuffleKey(applicationId, shuffleId)
+ val shuffleKey = Utils.makeShuffleKey(appUniqueId, shuffleId)
val parallelism = Math.min(Math.max(1, slotsToDestroy.size()), conf.clientRpcMaxParallelism)
ThreadUtils.parmap(
slotsToDestroy.asScala,
@@ -1040,7 +1012,7 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
commitManager.removeExpiredShuffle(shuffleId)
changePartitionManager.removeExpiredShuffle(shuffleId)
val unregisterShuffleResponse = requestMasterUnregisterShuffle(
- UnregisterShuffle(appId, shuffleId, RssHARetryClient.genRequestId()))
+ UnregisterShuffle(appUniqueId, shuffleId, RssHARetryClient.genRequestId()))
// if unregister shuffle not success, wait next turn
if (StatusCode.SUCCESS == Utils.toStatusCode(unregisterShuffleResponse.getStatus)) {
unregisterShuffleTime.remove(shuffleId)
@@ -1050,12 +1022,11 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
}
private def requestMasterRequestSlotsWithRetry(
- applicationId: String,
shuffleId: Int,
ids: util.ArrayList[Integer]): RequestSlotsResponse = {
val req =
RequestSlots(
- applicationId,
+ appUniqueId,
shuffleId,
ids,
lifecycleHost,
diff --git a/client/src/main/scala/org/apache/celeborn/client/ReleasePartitionManager.scala b/client/src/main/scala/org/apache/celeborn/client/ReleasePartitionManager.scala
index f8ae855f2..07d1ad7ff 100644
--- a/client/src/main/scala/org/apache/celeborn/client/ReleasePartitionManager.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/ReleasePartitionManager.scala
@@ -30,7 +30,6 @@ import org.apache.celeborn.common.protocol.message.ControlMessages.WorkerResourc
import org.apache.celeborn.common.util.{JavaUtils, ThreadUtils}
class ReleasePartitionManager(
- appId: String,
val conf: CelebornConf,
lifecycleManager: LifecycleManager)
extends Logging {
@@ -84,7 +83,6 @@ class ReleasePartitionManager(
if (!destroyResource.isEmpty) {
lifecycleManager.destroySlotsWithRetry(
- appId,
shuffleId,
destroyResource)
logTrace(s"Destroyed partition resource for shuffle $shuffleId $destroyResource")
@@ -127,7 +125,7 @@ class ReleasePartitionManager(
}
if (!destroyResource.isEmpty) {
- lifecycleManager.destroySlotsWithRetry(appId, shuffleId, destroyResource)
+ lifecycleManager.destroySlotsWithRetry(shuffleId, destroyResource)
logTrace(
s"Destroyed partition resource for partition $shuffleId-$partitionId, $destroyResource")
}
diff --git a/client/src/main/scala/org/apache/celeborn/client/commit/CommitHandler.scala b/client/src/main/scala/org/apache/celeborn/client/commit/CommitHandler.scala
index 1b9a9e5f2..d11c8eff0 100644
--- a/client/src/main/scala/org/apache/celeborn/client/commit/CommitHandler.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/commit/CommitHandler.scala
@@ -44,7 +44,7 @@ case class CommitResult(
commitFilesFailedWorkers: ShuffleFailedWorkers)
abstract class CommitHandler(
- appId: String,
+ appUniqueId: String,
conf: CelebornConf,
committedPartitionInfo: CommittedPartitionInfo,
workerStatusTracker: WorkerStatusTracker) extends Logging {
@@ -227,7 +227,7 @@ abstract class CommitHandler(
}
commitFiles(
- appId,
+ appUniqueId,
shuffleId,
shuffleCommittedInfo,
worker,
@@ -437,7 +437,7 @@ abstract class CommitHandler(
shuffleId: Int,
failedMasters: util.Map[String, WorkerInfo],
failedSlaves: util.Map[String, WorkerInfo]): Boolean = {
- val shuffleKey = Utils.makeShuffleKey(appId, shuffleId)
+ val shuffleKey = Utils.makeShuffleKey(appUniqueId, shuffleId)
if (!pushReplicateEnabled && failedMasters.size() != 0) {
val msg =
failedMasters.asScala.map {
diff --git a/client/src/main/scala/org/apache/celeborn/client/commit/ReducePartitionCommitHandler.scala b/client/src/main/scala/org/apache/celeborn/client/commit/ReducePartitionCommitHandler.scala
index 18214b51c..1ed1e570e 100644
--- a/client/src/main/scala/org/apache/celeborn/client/commit/ReducePartitionCommitHandler.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/commit/ReducePartitionCommitHandler.scala
@@ -47,12 +47,12 @@ import org.apache.celeborn.common.util.JavaUtils
* @see [[org.apache.celeborn.common.protocol.PartitionType.REDUCE]]
*/
class ReducePartitionCommitHandler(
- appId: String,
+ appUniqueId: String,
conf: CelebornConf,
shuffleAllocatedWorkers: ShuffleAllocatedWorkers,
committedPartitionInfo: CommittedPartitionInfo,
workerStatusTracker: WorkerStatusTracker)
- extends CommitHandler(appId, conf, committedPartitionInfo, workerStatusTracker)
+ extends CommitHandler(appUniqueId, conf, committedPartitionInfo, workerStatusTracker)
with Logging {
private val getReducerFileGroupRequest =
diff --git a/client/src/test/java/org/apache/celeborn/client/DummyShuffleClient.java b/client/src/test/java/org/apache/celeborn/client/DummyShuffleClient.java
index 896ef1a26..9d1cf27d3 100644
--- a/client/src/test/java/org/apache/celeborn/client/DummyShuffleClient.java
+++ b/client/src/test/java/org/apache/celeborn/client/DummyShuffleClient.java
@@ -63,7 +63,6 @@ public class DummyShuffleClient extends ShuffleClient {
@Override
public int pushData(
- String applicationId,
int shuffleId,
int mapId,
int attemptId,
@@ -83,7 +82,6 @@ public class DummyShuffleClient extends ShuffleClient {
@Override
public int mergeData(
- String applicationId,
int shuffleId,
int mapId,
int attemptId,
@@ -99,44 +97,32 @@ public class DummyShuffleClient extends ShuffleClient {
}
@Override
- public void pushMergedData(String applicationId, int shuffleId, int mapId, int attemptId) {}
+ public void pushMergedData(int shuffleId, int mapId, int attemptId) {}
@Override
- public void mapperEnd(
- String applicationId, int shuffleId, int mapId, int attemptId, int numMappers) {}
+ public void mapperEnd(int shuffleId, int mapId, int attemptId, int numMappers) {}
@Override
public void mapPartitionMapperEnd(
- String applicationId,
- int shuffleId,
- int mapId,
- int attemptId,
- int numMappers,
- int partitionId)
+ int shuffleId, int mapId, int attemptId, int numMappers, int partitionId)
throws IOException {}
@Override
- public void cleanup(String applicationId, int shuffleId, int mapId, int attemptId) {}
+ public void cleanup(int shuffleId, int mapId, int attemptId) {}
@Override
public RssInputStream readPartition(
- String applicationId,
- int shuffleId,
- int partitionId,
- int attemptNumber,
- int startMapIndex,
- int endMapIndex) {
+ int shuffleId, int partitionId, int attemptNumber, int startMapIndex, int endMapIndex) {
return null;
}
@Override
- public RssInputStream readPartition(
- String applicationId, int shuffleId, int partitionId, int attemptNumber) {
+ public RssInputStream readPartition(int shuffleId, int partitionId, int attemptNumber) {
return null;
}
@Override
- public boolean unregisterShuffle(String applicationId, int shuffleId, boolean isDriver) {
+ public boolean unregisterShuffle(int shuffleId, boolean isDriver) {
return false;
}
@@ -151,13 +137,13 @@ public class DummyShuffleClient extends ShuffleClient {
@Override
public PartitionLocation registerMapPartitionTask(
- String appId, int shuffleId, int numMappers, int mapId, int attemptId, int partitionId) {
+ int shuffleId, int numMappers, int mapId, int attemptId, int partitionId) {
return null;
}
@Override
public ConcurrentHashMap<Integer, PartitionLocation> getPartitionLocation(
- String applicationId, int shuffleId, int numMappers, int numPartitions) {
+ int shuffleId, int numMappers, int numPartitions) {
return reducePartitionMap.get(shuffleId);
}
diff --git a/client/src/test/java/org/apache/celeborn/client/ShuffleClientBaseSuiteJ.java b/client/src/test/java/org/apache/celeborn/client/ShuffleClientBaseSuiteJ.java
index b81da9aa1..6be02e151 100644
--- a/client/src/test/java/org/apache/celeborn/client/ShuffleClientBaseSuiteJ.java
+++ b/client/src/test/java/org/apache/celeborn/client/ShuffleClientBaseSuiteJ.java
@@ -85,12 +85,12 @@ public abstract class ShuffleClientBaseSuiteJ {
conf.set(CelebornConf.SHUFFLE_COMPRESSION_CODEC().key(), codec.name());
conf.set(CelebornConf.CLIENT_PUSH_RETRY_THREADS().key(), "1");
conf.set(CelebornConf.CLIENT_PUSH_BUFFER_MAX_SIZE().key(), "1K");
- shuffleClient = new ShuffleClientImpl(conf, new UserIdentifier("mock", "mock"));
+ shuffleClient =
+ new ShuffleClientImpl(TEST_APPLICATION_ID, conf, new UserIdentifier("mock", "mock"));
masterLocation.setPeer(slaveLocation);
when(endpointRef.askSync(
- ControlMessages.RegisterShuffle$.MODULE$.apply(
- TEST_APPLICATION_ID, TEST_SHUFFLE_ID, 1, 1),
+ ControlMessages.RegisterShuffle$.MODULE$.apply(TEST_SHUFFLE_ID, 1, 1),
ClassTag$.MODULE$.apply(PbRegisterShuffleResponse.class)))
.thenAnswer(
t ->
diff --git a/client/src/test/java/org/apache/celeborn/client/ShuffleClientSuiteJ.java b/client/src/test/java/org/apache/celeborn/client/ShuffleClientSuiteJ.java
index a34ca413d..3a380e92c 100644
--- a/client/src/test/java/org/apache/celeborn/client/ShuffleClientSuiteJ.java
+++ b/client/src/test/java/org/apache/celeborn/client/ShuffleClientSuiteJ.java
@@ -95,7 +95,6 @@ public class ShuffleClientSuiteJ {
int pushDataLen =
shuffleClient.pushData(
- TEST_APPLICATION_ID,
TEST_SHUFFLE_ID,
TEST_ATTEMPT_ID,
TEST_ATTEMPT_ID,
@@ -121,7 +120,6 @@ public class ShuffleClientSuiteJ {
int mergeSize =
shuffleClient.mergeData(
- TEST_APPLICATION_ID,
TEST_SHUFFLE_ID,
TEST_ATTEMPT_ID,
TEST_ATTEMPT_ID,
@@ -137,7 +135,6 @@ public class ShuffleClientSuiteJ {
final int compressedTotalSize = compressor.getCompressedTotalSize();
shuffleClient.mergeData(
- TEST_APPLICATION_ID,
TEST_SHUFFLE_ID,
TEST_ATTEMPT_ID,
TEST_ATTEMPT_ID,
@@ -153,7 +150,6 @@ public class ShuffleClientSuiteJ {
byte[] buf1k = RandomStringUtils.random(4000).getBytes(StandardCharsets.UTF_8);
int largeMergeSize =
shuffleClient.mergeData(
- TEST_APPLICATION_ID,
TEST_SHUFFLE_ID,
TEST_ATTEMPT_ID,
TEST_ATTEMPT_ID,
@@ -177,7 +173,8 @@ public class ShuffleClientSuiteJ {
conf.set(CelebornConf.SHUFFLE_COMPRESSION_CODEC().key(), codec.name());
conf.set(CelebornConf.CLIENT_PUSH_RETRY_THREADS().key(), "1");
conf.set(CelebornConf.CLIENT_PUSH_BUFFER_MAX_SIZE().key(), "1K");
- shuffleClient = new ShuffleClientImpl(conf, new UserIdentifier("mock", "mock"));
+ shuffleClient =
+ new ShuffleClientImpl(TEST_APPLICATION_ID, conf, new UserIdentifier("mock", "mock"));
masterLocation.setPeer(slaveLocation);
when(endpointRef.askSync(any(), any(), any()))
diff --git a/client/src/test/java/org/apache/celeborn/client/write/DataPushQueueSuitJ.java b/client/src/test/java/org/apache/celeborn/client/write/DataPushQueueSuitJ.java
index ff7ab1c13..0b69d62d7 100644
--- a/client/src/test/java/org/apache/celeborn/client/write/DataPushQueueSuitJ.java
+++ b/client/src/test/java/org/apache/celeborn/client/write/DataPushQueueSuitJ.java
@@ -82,7 +82,6 @@ public class DataPushQueueSuitJ {
CelebornConf conf = new CelebornConf();
conf.set(CelebornConf.CLIENT_PUSH_MAX_REQS_IN_FLIGHT().key(), "2");
- String app = "APP-1";
int shuffleId = 0;
int mapId = 0;
int attemptId = 0;
@@ -98,7 +97,6 @@ public class DataPushQueueSuitJ {
}
DataPusher dataPusher =
new DataPusher(
- app,
shuffleId,
mapId,
attemptId,
diff --git a/client/src/test/scala/org/apache/celeborn/client/WithShuffleClientSuite.scala b/client/src/test/scala/org/apache/celeborn/client/WithShuffleClientSuite.scala
index 06b8e0220..3563f73c2 100644
--- a/client/src/test/scala/org/apache/celeborn/client/WithShuffleClientSuite.scala
+++ b/client/src/test/scala/org/apache/celeborn/client/WithShuffleClientSuite.scala
@@ -57,12 +57,12 @@ trait WithShuffleClientSuite extends CelebornFunSuite {
prepareService()
shuffleId = 1
var location =
- shuffleClient.registerMapPartitionTask(APP, shuffleId, numMappers, mapId, attemptId, 1)
+ shuffleClient.registerMapPartitionTask(shuffleId, numMappers, mapId, attemptId, 1)
Assert.assertEquals(location.getId, 1)
// retry register
location =
- shuffleClient.registerMapPartitionTask(APP, shuffleId, numMappers, mapId, attemptId, 1)
+ shuffleClient.registerMapPartitionTask(shuffleId, numMappers, mapId, attemptId, 1)
Assert.assertEquals(location.getId, 1)
// check all allocated slots
@@ -73,13 +73,12 @@ trait WithShuffleClientSuite extends CelebornFunSuite {
// another mapId
location =
- shuffleClient.registerMapPartitionTask(APP, shuffleId, numMappers, mapId + 1, attemptId, 2)
+ shuffleClient.registerMapPartitionTask(shuffleId, numMappers, mapId + 1, attemptId, 2)
Assert.assertEquals(location.getId, 2)
// another mapId with another attemptId
location =
shuffleClient.registerMapPartitionTask(
- APP,
shuffleId,
numMappers,
mapId + 1,
@@ -142,39 +141,38 @@ trait WithShuffleClientSuite extends CelebornFunSuite {
Assert.assertEquals(shuffleClient.getReduceFileGroupsMap.size(), 0)
// reduce normal empty RssInputStream
- var stream = shuffleClient.readPartition(APP, shuffleId, 1, 1)
+ var stream = shuffleClient.readPartition(shuffleId, 1, 1)
Assert.assertEquals(stream.read(), -1)
// reduce normal null partition for RssInputStream
- stream = shuffleClient.readPartition(APP, shuffleId, 3, 1)
+ stream = shuffleClient.readPartition(shuffleId, 3, 1)
Assert.assertEquals(stream.read(), -1)
}
private def prepareService(): Unit = {
lifecycleManager = new LifecycleManager(APP, celebornConf)
- shuffleClient = new ShuffleClientImpl(celebornConf, userIdentifier)
+ shuffleClient = new ShuffleClientImpl(APP, celebornConf, userIdentifier)
shuffleClient.setupMetaServiceRef(lifecycleManager.self)
}
private def registerAndFinishPartition(): Unit = {
- shuffleClient.registerMapPartitionTask(APP, shuffleId, numMappers, mapId, attemptId, 1)
- shuffleClient.registerMapPartitionTask(APP, shuffleId, numMappers, mapId + 1, attemptId, 2)
- shuffleClient.registerMapPartitionTask(APP, shuffleId, numMappers, mapId + 2, attemptId, 3)
+ shuffleClient.registerMapPartitionTask(shuffleId, numMappers, mapId, attemptId, 1)
+ shuffleClient.registerMapPartitionTask(shuffleId, numMappers, mapId + 1, attemptId, 2)
+ shuffleClient.registerMapPartitionTask(shuffleId, numMappers, mapId + 2, attemptId, 3)
// task number incr to numMappers + 1
- shuffleClient.registerMapPartitionTask(APP, shuffleId, numMappers, mapId, attemptId + 1, 9)
- shuffleClient.mapPartitionMapperEnd(APP, shuffleId, mapId, attemptId, numMappers, 1)
+ shuffleClient.registerMapPartitionTask(shuffleId, numMappers, mapId, attemptId + 1, 9)
+ shuffleClient.mapPartitionMapperEnd(shuffleId, mapId, attemptId, numMappers, 1)
// retry
- shuffleClient.mapPartitionMapperEnd(APP, shuffleId, mapId, attemptId, numMappers, 1)
+ shuffleClient.mapPartitionMapperEnd(shuffleId, mapId, attemptId, numMappers, 1)
// another attempt
shuffleClient.mapPartitionMapperEnd(
- APP,
shuffleId,
mapId,
attemptId + 1,
numMappers,
9)
// another mapper
- shuffleClient.mapPartitionMapperEnd(APP, shuffleId, mapId + 1, attemptId, numMappers, mapId + 1)
+ shuffleClient.mapPartitionMapperEnd(shuffleId, mapId + 1, attemptId, numMappers, mapId + 1)
}
}
diff --git a/common/src/main/proto/TransportMessages.proto b/common/src/main/proto/TransportMessages.proto
index 31c201e65..731c6a81a 100644
--- a/common/src/main/proto/TransportMessages.proto
+++ b/common/src/main/proto/TransportMessages.proto
@@ -154,19 +154,17 @@ message PbHeartbeatResponse {
}
message PbRegisterShuffle {
- string applicationId = 1;
- int32 shuffleId = 2;
- int32 numMapppers = 3;
- int32 numPartitions = 4;
+ int32 shuffleId = 1;
+ int32 numMapppers = 2;
+ int32 numPartitions = 3;
}
message PbRegisterMapPartitionTask {
- string applicationId = 1;
- int32 shuffleId = 2;
- int32 numMappers = 3;
- int32 mapId = 4;
- int32 attemptId = 5;
- int32 partitionId = 6;
+ int32 shuffleId = 1;
+ int32 numMappers = 2;
+ int32 mapId = 3;
+ int32 attemptId = 4;
+ int32 partitionId = 5;
}
message PbRegisterShuffleResponse {
@@ -208,14 +206,13 @@ message PbRequestSlotsResponse {
}
message PbRevive {
- string applicationId = 1;
- int32 shuffleId = 2;
- int32 mapId = 3;
- int32 attemptId = 4;
- int32 partitionId = 5;
- int32 epoch = 6;
- PbPartitionLocation oldPartition = 7;
- int32 status = 8;
+ int32 shuffleId = 1;
+ int32 mapId = 2;
+ int32 attemptId = 3;
+ int32 partitionId = 4;
+ int32 epoch = 5;
+ PbPartitionLocation oldPartition = 6;
+ int32 status = 7;
}
message PbChangeLocationResponse {
@@ -225,20 +222,18 @@ message PbChangeLocationResponse {
}
message PbPartitionSplit {
- string applicationId = 1;
- int32 shuffleId = 2;
- int32 partitionId = 3;
- int32 epoch = 4;
- PbPartitionLocation oldPartition = 5;
+ int32 shuffleId = 1;
+ int32 partitionId = 2;
+ int32 epoch = 3;
+ PbPartitionLocation oldPartition = 4;
}
message PbMapperEnd {
- string applicationId = 1;
- int32 shuffleId = 2;
- int32 mapId = 3;
- int32 attemptId = 4;
- int32 numMappers = 5;
- int32 partitionId = 6;
+ int32 shuffleId = 1;
+ int32 mapId = 2;
+ int32 attemptId = 3;
+ int32 numMappers = 4;
+ int32 partitionId = 5;
}
message PbMapperEndResponse {
@@ -246,8 +241,7 @@ message PbMapperEndResponse {
}
message PbGetReducerFileGroup {
- string applicationId = 1;
- int32 shuffleId = 2;
+ int32 shuffleId = 1;
}
message PbGetReducerFileGroupResponse {
@@ -412,8 +406,7 @@ message PbWorkerLostResponse {
}
message PbStageEnd {
- string applicationId = 1;
- int32 shuffleId = 2;
+ int32 shuffleId = 1;
}
message PbStageEndResponse {
diff --git a/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala b/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala
index 370bfdca3..9bbbda7c4 100644
--- a/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala
@@ -119,12 +119,10 @@ object ControlMessages extends Logging {
object RegisterShuffle {
def apply(
- appId: String,
shuffleId: Int,
numMappers: Int,
numPartitions: Int): PbRegisterShuffle =
PbRegisterShuffle.newBuilder()
- .setApplicationId(appId)
.setShuffleId(shuffleId)
.setNumMapppers(numMappers)
.setNumPartitions(numPartitions)
@@ -133,14 +131,12 @@ object ControlMessages extends Logging {
object RegisterMapPartitionTask {
def apply(
- appId: String,
shuffleId: Int,
numMappers: Int,
mapId: Int,
attemptId: Int,
partitionId: Int): PbRegisterMapPartitionTask =
PbRegisterMapPartitionTask.newBuilder()
- .setApplicationId(appId)
.setShuffleId(shuffleId)
.setNumMappers(numMappers)
.setMapId(mapId)
@@ -189,7 +185,6 @@ object ControlMessages extends Logging {
object Revive {
def apply(
- appId: String,
shuffleId: Int,
mapId: Int,
attemptId: Int,
@@ -198,7 +193,7 @@ object ControlMessages extends Logging {
oldPartition: PartitionLocation,
cause: StatusCode): PbRevive = {
val builder = PbRevive.newBuilder()
- builder.setApplicationId(appId)
+ builder
.setShuffleId(shuffleId)
.setMapId(mapId)
.setAttemptId(attemptId)
@@ -214,13 +209,11 @@ object ControlMessages extends Logging {
object PartitionSplit {
def apply(
- appId: String,
shuffleId: Int,
partitionId: Int,
epoch: Int,
oldPartition: PartitionLocation): PbPartitionSplit =
PbPartitionSplit.newBuilder()
- .setApplicationId(appId)
.setShuffleId(shuffleId)
.setPartitionId(partitionId)
.setEpoch(epoch)
@@ -244,7 +237,6 @@ object ControlMessages extends Logging {
}
case class MapperEnd(
- applicationId: String,
shuffleId: Int,
mapId: Int,
attemptId: Int,
@@ -254,7 +246,7 @@ object ControlMessages extends Logging {
case class MapperEndResponse(status: StatusCode) extends MasterMessage
- case class GetReducerFileGroup(applicationId: String, shuffleId: Int) extends MasterMessage
+ case class GetReducerFileGroup(shuffleId: Int) extends MasterMessage
// util.Set[String] -> util.Set[Path.toString]
// Path can't be serialized
@@ -289,7 +281,7 @@ object ControlMessages extends Logging {
.build()
}
- case class StageEnd(applicationId: String, shuffleId: Int) extends MasterMessage
+ case class StageEnd(shuffleId: Int) extends MasterMessage
case class StageEndResponse(status: StatusCode)
extends MasterMessage
@@ -542,9 +534,8 @@ object ControlMessages extends Logging {
case pb: PbChangeLocationResponse =>
new TransportMessage(MessageType.CHANGE_LOCATION_RESPONSE, pb.toByteArray)
- case MapperEnd(applicationId, shuffleId, mapId, attemptId, numMappers, partitionId) =>
+ case MapperEnd(shuffleId, mapId, attemptId, numMappers, partitionId) =>
val payload = PbMapperEnd.newBuilder()
- .setApplicationId(applicationId)
.setShuffleId(shuffleId)
.setMapId(mapId)
.setAttemptId(attemptId)
@@ -559,9 +550,9 @@ object ControlMessages extends Logging {
.build().toByteArray
new TransportMessage(MessageType.MAPPER_END_RESPONSE, payload)
- case GetReducerFileGroup(applicationId, shuffleId) =>
+ case GetReducerFileGroup(shuffleId) =>
val payload = PbGetReducerFileGroup.newBuilder()
- .setApplicationId(applicationId).setShuffleId(shuffleId)
+ .setShuffleId(shuffleId)
.build().toByteArray
new TransportMessage(MessageType.GET_REDUCER_FILE_GROUP, payload)
@@ -587,9 +578,8 @@ object ControlMessages extends Logging {
case pb: PbWorkerLostResponse =>
new TransportMessage(MessageType.WORKER_LOST_RESPONSE, pb.toByteArray)
- case StageEnd(applicationId, shuffleId) =>
+ case StageEnd(shuffleId) =>
val payload = PbStageEnd.newBuilder()
- .setApplicationId(applicationId)
.setShuffleId(shuffleId)
.build().toByteArray
new TransportMessage(MessageType.STAGE_END, payload)
@@ -918,7 +908,6 @@ object ControlMessages extends Logging {
case MAPPER_END =>
val pbMapperEnd = PbMapperEnd.parseFrom(message.getPayload)
MapperEnd(
- pbMapperEnd.getApplicationId,
pbMapperEnd.getShuffleId,
pbMapperEnd.getMapId,
pbMapperEnd.getAttemptId,
@@ -932,7 +921,6 @@ object ControlMessages extends Logging {
case GET_REDUCER_FILE_GROUP =>
val pbGetReducerFileGroup = PbGetReducerFileGroup.parseFrom(message.getPayload)
GetReducerFileGroup(
- pbGetReducerFileGroup.getApplicationId,
pbGetReducerFileGroup.getShuffleId)
case GET_REDUCER_FILE_GROUP_RESPONSE =>
@@ -1144,7 +1132,7 @@ object ControlMessages extends Logging {
case STAGE_END =>
val pbStageEnd = PbStageEnd.parseFrom(message.getPayload)
- StageEnd(pbStageEnd.getApplicationId, pbStageEnd.getShuffleId)
+ StageEnd(pbStageEnd.getShuffleId)
case PARTITION_SPLIT =>
PbPartitionSplit.parseFrom(message.getPayload)
diff --git a/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala b/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
index f5fa1bfe3..dd5dae67e 100644
--- a/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
@@ -613,8 +613,8 @@ object Utils extends Logging {
(partitionId, epoch)
}
- def makeReducerKey(applicationId: String, shuffleId: Int, partitionId: Int): String = {
- s"$applicationId-$shuffleId-$partitionId"
+ def makeReducerKey(shuffleId: Int, partitionId: Int): String = {
+ s"shuffleId-$partitionId"
}
def makeMapKey(applicationId: String, shuffleId: Int, mapId: Int, attemptId: Int): String = {
diff --git a/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala b/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala
index 566392b2a..23f9ce368 100644
--- a/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala
+++ b/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala
@@ -99,7 +99,7 @@ class UtilsSuite extends CelebornFunSuite {
}
test("MapperEnd class convert with pb") {
- val mapperEnd = MapperEnd("application1", 1, 1, 1, 2, 1)
+ val mapperEnd = MapperEnd(1, 1, 1, 2, 1)
val mapperEndTrans =
Utils.fromTransportMessage(Utils.toTransportMessage(mapperEnd)).asInstanceOf[MapperEnd]
assert(mapperEnd == mapperEndTrans)
diff --git a/tests/flink-it/src/test/scala/org/apache/celeborn/tests/flink/HeartbeatTest.scala b/tests/flink-it/src/test/scala/org/apache/celeborn/tests/flink/HeartbeatTest.scala
index 1f7511814..0f7a065a2 100644
--- a/tests/flink-it/src/test/scala/org/apache/celeborn/tests/flink/HeartbeatTest.scala
+++ b/tests/flink-it/src/test/scala/org/apache/celeborn/tests/flink/HeartbeatTest.scala
@@ -33,6 +33,7 @@ class HeartbeatTest extends AnyFunSuite with Logging with MiniClusterFeature wit
val (_, clientConf) = getTestHeartbeatFromWorker2ClientConf()
val flinkShuffleClientImpl =
new FlinkShuffleClientImpl(
+ "",
"",
0,
System.currentTimeMillis(),
@@ -47,6 +48,7 @@ class HeartbeatTest extends AnyFunSuite with Logging with MiniClusterFeature wit
val (_, clientConf) = getTestHeartbeatFromWorker2ClientWithNoHeartbeatConf()
val flinkShuffleClientImpl =
new FlinkShuffleClientImpl(
+ "",
"",
0,
System.currentTimeMillis(),
@@ -61,6 +63,7 @@ class HeartbeatTest extends AnyFunSuite with Logging with MiniClusterFeature wit
val (_, clientConf) = getTestHeartbeatFromWorker2ClientWithCloseChannelConf()
val flinkShuffleClientImpl =
new FlinkShuffleClientImpl(
+ "",
"",
0,
System.currentTimeMillis(),
diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/client/ShuffleClientSuite.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/client/ShuffleClientSuite.scala
index a38cece56..624b7101c 100644
--- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/client/ShuffleClientSuite.scala
+++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/client/ShuffleClientSuite.scala
@@ -47,13 +47,13 @@ class ShuffleClientSuite extends WithShuffleClientSuite with MiniClusterFeature
val lifecycleManager: LifecycleManager = new LifecycleManager(APP, celebornConf)
val shuffleClient: ShuffleClientImpl = {
- val client = new ShuffleClientImpl(celebornConf, userIdentifier)
+ val client = new ShuffleClientImpl(APP, celebornConf, userIdentifier)
client.setupMetaServiceRef(lifecycleManager.self)
client
}
assertThrows[IOException] {
- shuffleClient.registerMapPartitionTask(APP, 1, 1, 0, 0, 1)
+ shuffleClient.registerMapPartitionTask(1, 1, 0, 0, 1)
}
lifecycleManager.stop()
diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/HeartbeatTest.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/HeartbeatTest.scala
index 23c278c42..5e167f18c 100644
--- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/HeartbeatTest.scala
+++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/HeartbeatTest.scala
@@ -39,21 +39,21 @@ class HeartbeatTest extends AnyFunSuite with Logging with MiniClusterFeature wit
test("celeborn spark heartbeat test - client <- worker") {
val (_, clientConf) = getTestHeartbeatFromWorker2ClientConf()
val shuffleClientImpl =
- new ShuffleClientImpl(clientConf, new UserIdentifier("1", "1"))
+ new ShuffleClientImpl("APP", clientConf, new UserIdentifier("1", "1"))
testHeartbeatFromWorker2Client(shuffleClientImpl.getDataClientFactory)
}
test("celeborn spark heartbeat test - client <- worker on heartbeat") {
val (_, clientConf) = getTestHeartbeatFromWorker2ClientWithNoHeartbeatConf()
val shuffleClientImpl =
- new ShuffleClientImpl(clientConf, new UserIdentifier("1", "1"))
+ new ShuffleClientImpl("APP", clientConf, new UserIdentifier("1", "1"))
testHeartbeatFromWorker2ClientWithNoHeartbeat(shuffleClientImpl.getDataClientFactory)
}
test("celeborn spark heartbeat test - client <- worker timeout") {
val (_, clientConf) = getTestHeartbeatFromWorker2ClientWithCloseChannelConf()
val shuffleClientImpl =
- new ShuffleClientImpl(clientConf, new UserIdentifier("1", "1"))
+ new ShuffleClientImpl("APP", clientConf, new UserIdentifier("1", "1"))
testHeartbeatFromWorker2ClientWithCloseChannel(shuffleClientImpl.getDataClientFactory)
}
}
diff --git a/worker/src/test/scala/org/apache/celeborn/service/deploy/cluster/ReadWriteTestBase.scala b/worker/src/test/scala/org/apache/celeborn/service/deploy/cluster/ReadWriteTestBase.scala
index 8bba3c682..0b53aef41 100644
--- a/worker/src/test/scala/org/apache/celeborn/service/deploy/cluster/ReadWriteTestBase.scala
+++ b/worker/src/test/scala/org/apache/celeborn/service/deploy/cluster/ReadWriteTestBase.scala
@@ -61,7 +61,7 @@ trait ReadWriteTestBase extends AnyFunSuite
.set(CelebornConf.CLIENT_PUSH_BUFFER_MAX_SIZE.key, "256K")
.set("celeborn.data.io.numConnectionsPerPeer", "1")
val lifecycleManager = new LifecycleManager(APP, clientConf)
- val shuffleClient = new ShuffleClientImpl(clientConf, UserIdentifier("mock", "mock"))
+ val shuffleClient = new ShuffleClientImpl(APP, clientConf, UserIdentifier("mock", "mock"))
shuffleClient.setupMetaServiceRef(lifecycleManager.self)
val STR1 = RandomStringUtils.random(1024)
@@ -69,31 +69,31 @@ trait ReadWriteTestBase extends AnyFunSuite
val OFFSET1 = 0
val LENGTH1 = DATA1.length
- val dataSize1 = shuffleClient.pushData(APP, 1, 0, 0, 0, DATA1, OFFSET1, LENGTH1, 1, 1)
+ val dataSize1 = shuffleClient.pushData(1, 0, 0, 0, DATA1, OFFSET1, LENGTH1, 1, 1)
logInfo(s"push data data size $dataSize1")
val STR2 = RandomStringUtils.random(32 * 1024)
val DATA2 = STR2.getBytes(StandardCharsets.UTF_8)
val OFFSET2 = 0
val LENGTH2 = DATA2.length
- val dataSize2 = shuffleClient.pushData(APP, 1, 0, 0, 0, DATA2, OFFSET2, LENGTH2, 1, 1)
+ val dataSize2 = shuffleClient.pushData(1, 0, 0, 0, DATA2, OFFSET2, LENGTH2, 1, 1)
logInfo("push data data size " + dataSize2)
val STR3 = RandomStringUtils.random(32 * 1024)
val DATA3 = STR3.getBytes(StandardCharsets.UTF_8)
val LENGTH3 = DATA3.length
- shuffleClient.mergeData(APP, 1, 0, 0, 0, DATA3, 0, LENGTH3, 1, 1)
+ shuffleClient.mergeData(1, 0, 0, 0, DATA3, 0, LENGTH3, 1, 1)
val STR4 = RandomStringUtils.random(16 * 1024)
val DATA4 = STR4.getBytes(StandardCharsets.UTF_8)
val LENGTH4 = DATA4.length
- shuffleClient.mergeData(APP, 1, 0, 0, 0, DATA4, 0, LENGTH4, 1, 1)
- shuffleClient.pushMergedData(APP, 1, 0, 0)
+ shuffleClient.mergeData(1, 0, 0, 0, DATA4, 0, LENGTH4, 1, 1)
+ shuffleClient.pushMergedData(1, 0, 0)
Thread.sleep(1000)
- shuffleClient.mapperEnd(APP, 1, 0, 0, 1)
+ shuffleClient.mapperEnd(1, 0, 0, 1)
- val inputStream = shuffleClient.readPartition(APP, 1, 0, 0)
+ val inputStream = shuffleClient.readPartition(1, 0, 0)
val outputStream = new ByteArrayOutputStream()
var b = inputStream.read()