[GitHub] [incubator-uniffle] zhengchenyu commented on issue #993: [Improvement][tez] Optimize the method of obtain the application attempt id.

["zhengchenyu (via GitHub)" <gi...@apache.org>]

zhengchenyu commented on issue #993:
URL: https://github.com/apache/incubator-uniffle/issues/993#issuecomment-1620047065

   The logical of tez local mode is similar to tez yarn mode,  so tez local mode is useful way in development mode.
   
   If wanna tez local mode run with uniffle, need below patch:
   
   ```
   ---
    .../org/apache/tez/client/LocalClient.java    | 26 ++++++++++++++++---
    1 file changed, 22 insertions(+), 4 deletions(-)
   
   diff --git a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
   index 6baea48..4612645 100644
   --- a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
   +++ b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
   @@ -45,10 +45,12 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
    import org.apache.hadoop.yarn.api.records.ApplicationReport;
    import org.apache.hadoop.yarn.api.records.YarnApplicationState;
    import org.apache.hadoop.yarn.util.SystemClock;
   +import org.apache.tez.common.ReflectionUtils;
    import org.apache.tez.common.TezCommonUtils;
    import org.apache.tez.common.TezUtilsInternal;
    import org.apache.tez.dag.api.TezConfiguration;
    import org.apache.tez.dag.api.TezException;
   +import org.apache.tez.dag.api.TezReflectionException;
    import org.apache.tez.dag.api.client.DAGClientHandler;
    import org.apache.tez.dag.api.records.DAGProtos;
    import org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto;
   @@ -312,7 +314,7 @@ public class LocalClient extends FrameworkClient {
              }
   
              // Construct, initialize, and start the DAGAppMaster
   -          ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 0);
   +          ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 1);
              ContainerId cId = ContainerId.newInstance(applicationAttemptId, 1);
              String currentHost = InetAddress.getLocalHost().getHostName();
              int nmPort = YarnConfiguration.DEFAULT_NM_PORT;
   @@ -360,9 +362,25 @@ public class LocalClient extends FrameworkClient {
            getPluginDescriptorInfo(conf, applicationAttemptId.getApplicationId().toString());
   
   
   -    return new DAGAppMaster(applicationAttemptId, cId, currentHost, nmPort, nmHttpPort,
   -        new SystemClock(), appSubmitTime, isSession, userDir, localDirs, logDirs,
   -        versionInfo.getVersion(), credentials, jobUserName, amPluginDescriptorProto);
   +    String className = this.conf.get("tez.local.appmaster.class", DAGAppMaster.class.getName());
   +    try {
   +      return ReflectionUtils.createClazzInstance(className,
   +          new Class[] {
   +              ApplicationAttemptId.class, ContainerId.class, String.class, int.class, int.class,
   +              Clock.class, long.class, boolean.class, String.class, String[].class, String[].class,
   +              String.class, Credentials.class, String.class, AMPluginDescriptorProto.class
   +          },
   +          new Object[] {
   +              applicationAttemptId, cId, currentHost, nmPort, nmHttpPort,
   +              new SystemClock(), appSubmitTime, isSession, userDir, localDirs, logDirs,
   +              versionInfo.getVersion(), credentials, jobUserName, amPluginDescriptorProto
   +          }
   +      );
   +    } catch (TezReflectionException e) {
   +      return new DAGAppMaster(applicationAttemptId, cId, currentHost, nmPort, nmHttpPort,
   +          new SystemClock(), appSubmitTime, isSession, userDir, localDirs, logDirs,
   +          versionInfo.getVersion(), credentials, jobUserName, amPluginDescriptorProto);
   +    }
      }
   
      private AMPluginDescriptorProto getPluginDescriptorInfo(Configuration conf,
   --
   ```
   
   And the tez local application command is:
   
   ```
   HADOOP_CLASSPATH=${TEZ_CONF_DIR}:${TEZ_HOME}/*:${TEZ_HOME}/lib/*:rss-client-tez-XXX-shaded.jar
   hadoop jar ${TEZ_HOME}/tez-examples-0.9.2.jar wordcount \
     -Dhadoop.security.authentication=simple \
     -local \
     /XXX/examples/tez-examples/data/wordcount/input \
     /XXX/examples/tez-examples/data/wordcount/output \
     2
   ```


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: dev-unsubscribe@uniffle.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org