You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by GitBox <gi...@apache.org> on 2022/03/01 09:16:22 UTC
[GitHub] [flink] bgeng777 commented on a change in pull request #18739: [FLINK-26030][yarn] Set FLINK_LIB_DIR to 'lib' under working dir in YARN containers
bgeng777 commented on a change in pull request #18739:
URL: https://github.com/apache/flink/pull/18739#discussion_r816581955
##########
File path: flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java
##########
@@ -830,4 +832,42 @@ private YarnClusterDescriptor createYarnClusterDescriptor(Configuration configur
.setYarnClusterInformationRetriever(() -> YARN_MAX_VCORES)
.build();
}
+
+ @Test
+ public void testGenerateApplicationMasterEnv() throws IOException {
+ final Configuration flinkConfig = new Configuration();
+ final File flinkHomeDir = temporaryFolder.newFolder();
+ final String fakeLocalFlinkJar = "./lib/flink_dist.jar";
+ final String fakeClassPath = fakeLocalFlinkJar + ":./usrlib/user.jar";
+ final ApplicationId appId = ApplicationId.newInstance(0, 0);
+ try (final YarnClusterDescriptor yarnClusterDescriptor =
+ createYarnClusterDescriptor(flinkConfig)) {
+ final YarnApplicationFileUploader yarnApplicationFileUploader =
+ YarnApplicationFileUploader.from(
+ FileSystem.get(new YarnConfiguration()),
+ new Path(flinkHomeDir.getPath()),
+ new ArrayList<>(),
+ appId,
+ DFSConfigKeys.DFS_REPLICATION_DEFAULT);
+ final Map<String, String> masterEnv =
+ yarnClusterDescriptor.generateApplicationMasterEnv(
+ yarnApplicationFileUploader,
+ fakeClassPath,
+ fakeLocalFlinkJar,
+ appId.toString());
+
+ Assert.assertEquals("./lib", masterEnv.get(ConfigConstants.ENV_FLINK_LIB_DIR));
Review comment:
Fixed in 0b12e096e97eff0fddc7a2e777f4cd40be5a4df3
##########
File path: flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
##########
@@ -1829,4 +1809,41 @@ public static void logDetachedClusterInformation(
yarnApplicationId,
yarnApplicationId);
}
+
+ @VisibleForTesting
+ Map<String, String> generateApplicationMasterEnv(
+ final YarnApplicationFileUploader fileUploader,
+ final String classPathStr,
+ final String localFlinkJarStr,
+ final String appIdStr)
+ throws IOException {
+ final Map<String, String> env = new HashMap<>();
+ // set YARN classpath
+ env.put(ENV_FLINK_CLASSPATH, classPathStr);
+ Utils.setupYarnClassPath(this.yarnConfiguration, env);
+ // https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md#identity-on-an-insecure-cluster-hadoop_user_name
+ env.put(
+ YarnConfigKeys.ENV_HADOOP_USER_NAME,
+ UserGroupInformation.getCurrentUser().getUserName());
+ // set user specified app master environment variables
+ env.putAll(
+ ConfigurationUtils.getPrefixedKeyValuePairs(
+ ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX,
+ this.flinkConfiguration));
+ // Set FLINK_LIB_DIR to `lib` folder under working dir in container
+ env.put(ENV_FLINK_LIB_DIR, Path.CUR_DIR + "/" + ConfigConstants.DEFAULT_FLINK_LIB_DIR);
Review comment:
Add in 0b12e096e97eff0fddc7a2e777f4cd40be5a4df3
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org