You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hive.apache.org by "László Bodor (Jira)" <ji...@apache.org> on 2022/11/11 08:25:00 UTC
[jira] [Updated] (HIVE-26727) Log all mr -> tez config overrides
[ https://issues.apache.org/jira/browse/HIVE-26727?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
László Bodor updated HIVE-26727:
--------------------------------
Description:
in TezSessionState.setupTezParamsBasedOnMR
{code}
private void setupTezParamsBasedOnMR(TezConfiguration conf) {
String env = conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV);
if (conf.get(MRJobConfig.MR_AM_ENV) != null) {
env = (env == null) ? conf.get(MRJobConfig.MR_AM_ENV) : env + "," + conf.get(MRJobConfig.MR_AM_ENV);
}
if (env != null) {
conf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_ENV, env);
}
conf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
org.apache.tez.mapreduce.hadoop.MRHelpers.getJavaOptsForMRAM(conf));
String queueName = conf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME);
conf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName); <---- NOT LOGGED AT ALL
int amMemMB = conf.getInt(MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB);
conf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, "" + amMemMB);<---- NOT LOGGED AT ALL
int amCores = conf.getInt(MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES);
conf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, "" + amCores);<---- NOT LOGGED AT ALL
conf.setIfUnset(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, ""<---- NOT LOGGED AT ALL
+ conf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));
conf.setIfUnset(TezConfiguration.TEZ_AM_VIEW_ACLS,<---- NOT LOGGED AT ALL
conf.get(MRJobConfig.JOB_ACL_VIEW_JOB, MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB));
conf.setIfUnset(TezConfiguration.TEZ_AM_MODIFY_ACLS,<---- NOT LOGGED AT ALL
conf.get(MRJobConfig.JOB_ACL_MODIFY_JOB, MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB));
// Refer to org.apache.tez.mapreduce.hadoop.MRHelpers.processDirectConversion.
ArrayList<Map<String, String>> maps = new ArrayList<Map<String, String>>(2);
maps.add(DeprecatedKeys.getMRToTezRuntimeParamMap());
maps.add(DeprecatedKeys.getMRToDAGParamMap());
boolean preferTez = true; // Can make this configurable.
for (Map<String, String> map : maps) {
for (Map.Entry<String, String> dep : map.entrySet()) {
if (conf.get(dep.getKey()) != null) {
// TODO Deprecation reason does not seem to reflect in the config ?
// The ordering is important in case of keys which are also deprecated.
// Unset will unset the deprecated keys and all its variants.
final String mrValue = conf.get(dep.getKey());
final String tezValue = conf.get(dep.getValue());
conf.unset(dep.getKey());
if (tezValue == null) {
conf.set(dep.getValue(), mrValue, "TRANSLATED_TO_TEZ");
} else if (!preferTez) {
conf.set(dep.getValue(), mrValue, "TRANSLATED_TO_TEZ_AND_MR_OVERRIDE");
}
LOG.info("Config: mr(unset):" + dep.getKey() + ", mr initial value=" <---- LOGGED PROPERLY
+ mrValue
+ ", tez(original):" + dep.getValue() + "=" + tezValue
+ ", tez(final):" + dep.getValue() + "=" + conf.get(dep.getValue()));
}
}
}
}
{code}
> Log all mr -> tez config overrides
> ----------------------------------
>
> Key: HIVE-26727
> URL: https://issues.apache.org/jira/browse/HIVE-26727
> Project: Hive
> Issue Type: Improvement
> Reporter: László Bodor
> Priority: Major
>
> in TezSessionState.setupTezParamsBasedOnMR
> {code}
> private void setupTezParamsBasedOnMR(TezConfiguration conf) {
> String env = conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV);
> if (conf.get(MRJobConfig.MR_AM_ENV) != null) {
> env = (env == null) ? conf.get(MRJobConfig.MR_AM_ENV) : env + "," + conf.get(MRJobConfig.MR_AM_ENV);
> }
> if (env != null) {
> conf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_ENV, env);
> }
> conf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
> org.apache.tez.mapreduce.hadoop.MRHelpers.getJavaOptsForMRAM(conf));
> String queueName = conf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME);
> conf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName); <---- NOT LOGGED AT ALL
> int amMemMB = conf.getInt(MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB);
> conf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, "" + amMemMB);<---- NOT LOGGED AT ALL
> int amCores = conf.getInt(MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES);
> conf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, "" + amCores);<---- NOT LOGGED AT ALL
> conf.setIfUnset(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, ""<---- NOT LOGGED AT ALL
> + conf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));
> conf.setIfUnset(TezConfiguration.TEZ_AM_VIEW_ACLS,<---- NOT LOGGED AT ALL
> conf.get(MRJobConfig.JOB_ACL_VIEW_JOB, MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB));
> conf.setIfUnset(TezConfiguration.TEZ_AM_MODIFY_ACLS,<---- NOT LOGGED AT ALL
> conf.get(MRJobConfig.JOB_ACL_MODIFY_JOB, MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB));
> // Refer to org.apache.tez.mapreduce.hadoop.MRHelpers.processDirectConversion.
> ArrayList<Map<String, String>> maps = new ArrayList<Map<String, String>>(2);
> maps.add(DeprecatedKeys.getMRToTezRuntimeParamMap());
> maps.add(DeprecatedKeys.getMRToDAGParamMap());
> boolean preferTez = true; // Can make this configurable.
> for (Map<String, String> map : maps) {
> for (Map.Entry<String, String> dep : map.entrySet()) {
> if (conf.get(dep.getKey()) != null) {
> // TODO Deprecation reason does not seem to reflect in the config ?
> // The ordering is important in case of keys which are also deprecated.
> // Unset will unset the deprecated keys and all its variants.
> final String mrValue = conf.get(dep.getKey());
> final String tezValue = conf.get(dep.getValue());
> conf.unset(dep.getKey());
> if (tezValue == null) {
> conf.set(dep.getValue(), mrValue, "TRANSLATED_TO_TEZ");
> } else if (!preferTez) {
> conf.set(dep.getValue(), mrValue, "TRANSLATED_TO_TEZ_AND_MR_OVERRIDE");
> }
> LOG.info("Config: mr(unset):" + dep.getKey() + ", mr initial value=" <---- LOGGED PROPERLY
> + mrValue
> + ", tez(original):" + dep.getValue() + "=" + tezValue
> + ", tez(final):" + dep.getValue() + "=" + conf.get(dep.getValue()));
> }
> }
> }
> }
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)