You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gy...@apache.org on 2021/09/23 08:23:36 UTC
[flink] branch master updated: [FLINK-23991][yarn] Specifying
yarn.staging-dir fail when staging scheme is different from default fs
scheme
This is an automated email from the ASF dual-hosted git repository.
gyfora pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 5bf8d64 [FLINK-23991][yarn] Specifying yarn.staging-dir fail when staging scheme is different from default fs scheme
5bf8d64 is described below
commit 5bf8d649627c801a5154bb78682308fcefe9210f
Author: Junfan Zhang <ju...@outlook.com>
AuthorDate: Fri Sep 10 11:59:28 2021 +0800
[FLINK-23991][yarn] Specifying yarn.staging-dir fail when staging scheme is different from default fs scheme
Closes #16994
---
.../apache/flink/yarn/YarnClusterDescriptor.java | 20 +++++++++-----
.../flink/yarn/YarnClusterDescriptorTest.java | 31 ++++++++++++++++++++++
2 files changed, 44 insertions(+), 7 deletions(-)
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
index 7bc07c5..af190f2 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
@@ -800,10 +800,12 @@ public class YarnClusterDescriptor implements ClusterDescriptor<ApplicationId> {
final List<Path> providedLibDirs =
Utils.getQualifiedRemoteSharedPaths(configuration, yarnConfiguration);
+ Path stagingDirPath = getStagingDir(fs);
+ FileSystem stagingDirFs = stagingDirPath.getFileSystem(yarnConfiguration);
final YarnApplicationFileUploader fileUploader =
YarnApplicationFileUploader.from(
- fs,
- getStagingDir(fs),
+ stagingDirFs,
+ stagingDirPath,
providedLibDirs,
appContext.getApplicationId(),
getFileReplication());
@@ -1250,15 +1252,19 @@ public class YarnClusterDescriptor implements ClusterDescriptor<ApplicationId> {
* Returns the configured remote target home directory if set, otherwise returns the default
* home directory.
*
- * @param fileSystem file system used
+ * @param defaultFileSystem default file system used
* @return the remote target home directory
*/
- private Path getStagingDir(FileSystem fileSystem) {
+ @VisibleForTesting
+ Path getStagingDir(FileSystem defaultFileSystem) throws IOException {
final String configuredStagingDir =
flinkConfiguration.getString(YarnConfigOptions.STAGING_DIRECTORY);
- return configuredStagingDir != null
- ? fileSystem.makeQualified(new Path(configuredStagingDir))
- : fileSystem.getHomeDirectory();
+ if (configuredStagingDir == null) {
+ return defaultFileSystem.getHomeDirectory();
+ }
+ FileSystem stagingDirFs =
+ new Path(configuredStagingDir).getFileSystem(defaultFileSystem.getConf());
+ return stagingDirFs.makeQualified(new Path(configuredStagingDir));
}
private int getFileReplication() {
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java
index 2e09279..1b8a8e2 100644
--- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java
@@ -36,6 +36,7 @@ import org.apache.flink.yarn.configuration.YarnConfigOptionsInternal;
import org.apache.flink.yarn.configuration.YarnDeploymentTarget;
import org.apache.flink.yarn.configuration.YarnLogConfigUtil;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -742,6 +743,36 @@ public class YarnClusterDescriptorTest extends TestLogger {
}
@Test
+ public void testGetStagingDirWithoutSpecifyingStagingDir() throws IOException {
+ try (final YarnClusterDescriptor yarnClusterDescriptor = createYarnClusterDescriptor()) {
+ YarnConfiguration yarnConfig = new YarnConfiguration();
+ yarnConfig.set("fs.defaultFS", "file://tmp");
+ FileSystem defaultFileSystem = FileSystem.get(yarnConfig);
+ Path stagingDir = yarnClusterDescriptor.getStagingDir(defaultFileSystem);
+ assertEquals("file", defaultFileSystem.getScheme());
+ assertEquals("file", stagingDir.getFileSystem(yarnConfig).getScheme());
+ }
+ }
+
+ @Test
+ public void testGetStagingDirWithSpecifyingStagingDir() throws IOException {
+ final Configuration flinkConfig = new Configuration();
+ flinkConfig.set(YarnConfigOptions.STAGING_DIRECTORY, "file:///tmp/path1");
+ try (final YarnClusterDescriptor yarnClusterDescriptor =
+ createYarnClusterDescriptor(flinkConfig)) {
+ YarnConfiguration yarnConfig = new YarnConfiguration();
+ yarnConfig.set("fs.defaultFS", "viewfs://hadoop-ns01");
+ yarnConfig.set("fs.viewfs.mounttable.hadoop-ns01.link./tmp", "file://tmp");
+ FileSystem defaultFileSystem = FileSystem.get(yarnConfig);
+
+ Path stagingDir = yarnClusterDescriptor.getStagingDir(defaultFileSystem);
+
+ assertEquals("viewfs", defaultFileSystem.getScheme());
+ assertEquals("file", stagingDir.getFileSystem(yarnConfig).getScheme());
+ }
+ }
+
+ @Test
public void testDeployApplicationClusterWithMultipleJarsSet() {
final Configuration flinkConfig = new Configuration();
flinkConfig.set(