You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gy...@apache.org on 2021/09/23 08:23:36 UTC

[flink] branch master updated: [FLINK-23991][yarn] Specifying yarn.staging-dir fail when staging scheme is different from default fs scheme

This is an automated email from the ASF dual-hosted git repository.

gyfora pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new 5bf8d64  [FLINK-23991][yarn] Specifying yarn.staging-dir fail when staging scheme is different from default fs scheme
5bf8d64 is described below

commit 5bf8d649627c801a5154bb78682308fcefe9210f
Author: Junfan Zhang <ju...@outlook.com>
AuthorDate: Fri Sep 10 11:59:28 2021 +0800

    [FLINK-23991][yarn] Specifying yarn.staging-dir fail when staging scheme is different from default fs scheme
    
    Closes #16994
---
 .../apache/flink/yarn/YarnClusterDescriptor.java   | 20 +++++++++-----
 .../flink/yarn/YarnClusterDescriptorTest.java      | 31 ++++++++++++++++++++++
 2 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
index 7bc07c5..af190f2 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
@@ -800,10 +800,12 @@ public class YarnClusterDescriptor implements ClusterDescriptor<ApplicationId> {
         final List<Path> providedLibDirs =
                 Utils.getQualifiedRemoteSharedPaths(configuration, yarnConfiguration);
 
+        Path stagingDirPath = getStagingDir(fs);
+        FileSystem stagingDirFs = stagingDirPath.getFileSystem(yarnConfiguration);
         final YarnApplicationFileUploader fileUploader =
                 YarnApplicationFileUploader.from(
-                        fs,
-                        getStagingDir(fs),
+                        stagingDirFs,
+                        stagingDirPath,
                         providedLibDirs,
                         appContext.getApplicationId(),
                         getFileReplication());
@@ -1250,15 +1252,19 @@ public class YarnClusterDescriptor implements ClusterDescriptor<ApplicationId> {
      * Returns the configured remote target home directory if set, otherwise returns the default
      * home directory.
      *
-     * @param fileSystem file system used
+     * @param defaultFileSystem default file system used
      * @return the remote target home directory
      */
-    private Path getStagingDir(FileSystem fileSystem) {
+    @VisibleForTesting
+    Path getStagingDir(FileSystem defaultFileSystem) throws IOException {
         final String configuredStagingDir =
                 flinkConfiguration.getString(YarnConfigOptions.STAGING_DIRECTORY);
-        return configuredStagingDir != null
-                ? fileSystem.makeQualified(new Path(configuredStagingDir))
-                : fileSystem.getHomeDirectory();
+        if (configuredStagingDir == null) {
+            return defaultFileSystem.getHomeDirectory();
+        }
+        FileSystem stagingDirFs =
+                new Path(configuredStagingDir).getFileSystem(defaultFileSystem.getConf());
+        return stagingDirFs.makeQualified(new Path(configuredStagingDir));
     }
 
     private int getFileReplication() {
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java
index 2e09279..1b8a8e2 100644
--- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java
@@ -36,6 +36,7 @@ import org.apache.flink.yarn.configuration.YarnConfigOptionsInternal;
 import org.apache.flink.yarn.configuration.YarnDeploymentTarget;
 import org.apache.flink.yarn.configuration.YarnLogConfigUtil;
 
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -742,6 +743,36 @@ public class YarnClusterDescriptorTest extends TestLogger {
     }
 
     @Test
+    public void testGetStagingDirWithoutSpecifyingStagingDir() throws IOException {
+        try (final YarnClusterDescriptor yarnClusterDescriptor = createYarnClusterDescriptor()) {
+            YarnConfiguration yarnConfig = new YarnConfiguration();
+            yarnConfig.set("fs.defaultFS", "file://tmp");
+            FileSystem defaultFileSystem = FileSystem.get(yarnConfig);
+            Path stagingDir = yarnClusterDescriptor.getStagingDir(defaultFileSystem);
+            assertEquals("file", defaultFileSystem.getScheme());
+            assertEquals("file", stagingDir.getFileSystem(yarnConfig).getScheme());
+        }
+    }
+
+    @Test
+    public void testGetStagingDirWithSpecifyingStagingDir() throws IOException {
+        final Configuration flinkConfig = new Configuration();
+        flinkConfig.set(YarnConfigOptions.STAGING_DIRECTORY, "file:///tmp/path1");
+        try (final YarnClusterDescriptor yarnClusterDescriptor =
+                createYarnClusterDescriptor(flinkConfig)) {
+            YarnConfiguration yarnConfig = new YarnConfiguration();
+            yarnConfig.set("fs.defaultFS", "viewfs://hadoop-ns01");
+            yarnConfig.set("fs.viewfs.mounttable.hadoop-ns01.link./tmp", "file://tmp");
+            FileSystem defaultFileSystem = FileSystem.get(yarnConfig);
+
+            Path stagingDir = yarnClusterDescriptor.getStagingDir(defaultFileSystem);
+
+            assertEquals("viewfs", defaultFileSystem.getScheme());
+            assertEquals("file", stagingDir.getFileSystem(yarnConfig).getScheme());
+        }
+    }
+
+    @Test
     public void testDeployApplicationClusterWithMultipleJarsSet() {
         final Configuration flinkConfig = new Configuration();
         flinkConfig.set(