You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ab...@apache.org on 2024/08/05 12:06:30 UTC

(tez) branch master updated: TEZ-4547: Add Tez AM JobID to the JobConf (#339) (Venkatasubrahmanian Narayanan reviewed by Laszlo Bodor)

This is an automated email from the ASF dual-hosted git repository.

abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git


The following commit(s) were added to refs/heads/master by this push:
     new 563b4947b TEZ-4547: Add Tez AM JobID to the JobConf (#339) (Venkatasubrahmanian Narayanan reviewed by Laszlo Bodor)
563b4947b is described below

commit 563b4947b53e9070e2e99cba91102e377f439ebc
Author: Venkatasubrahmanian Narayanan <10...@users.noreply.github.com>
AuthorDate: Mon Aug 5 05:06:25 2024 -0700

    TEZ-4547: Add Tez AM JobID to the JobConf (#339) (Venkatasubrahmanian Narayanan reviewed by Laszlo Bodor)
---
 .../tez/runtime/api/OutputCommitterContext.java    |  2 ++
 .../app/dag/impl/OutputCommitterContextImpl.java   | 18 +++++++++++++-----
 .../apache/tez/dag/app/dag/impl/VertexImpl.java    |  6 ++++--
 .../tez/mapreduce/committer/MROutputCommitter.java |  2 ++
 .../org/apache/tez/mapreduce/common/Utils.java     | 12 +++++++++++-
 .../apache/tez/mapreduce/hadoop/MRJobConfig.java   |  5 +++++
 .../org/apache/tez/mapreduce/output/MROutput.java  |  2 ++
 .../apache/tez/mapreduce/output/TestMROutput.java  | 22 ++++++++++++++++++++++
 8 files changed, 61 insertions(+), 8 deletions(-)

diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java
index d254a6a99..5b8906d52 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java
@@ -78,4 +78,6 @@ public interface OutputCommitterContext {
    */
   public int getVertexIndex();
 
+  public int getDagIdentifier();
+
 }
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java
index dc8951495..06be989b9 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java
@@ -34,14 +34,16 @@ public class OutputCommitterContextImpl implements OutputCommitterContext {
   private final String dagName;
   private final String vertexName;
   private final int vertexIdx;
+  private final int dagIdentifier;
   private final RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output;
 
   public OutputCommitterContextImpl(ApplicationId applicationId,
-      int dagAttemptNumber,
-      String dagName,
-      String vertexName,
-      RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output,
-      int vertexIdx) {
+                                    int dagAttemptNumber,
+                                    String dagName,
+                                    String vertexName,
+                                    int dagIdentifier,
+                                    int vertexIdx,
+                                    RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output) {
     Objects.requireNonNull(applicationId, "applicationId is null");
     Objects.requireNonNull(dagName, "dagName is null");
     Objects.requireNonNull(vertexName, "vertexName is null");
@@ -52,6 +54,7 @@ public class OutputCommitterContextImpl implements OutputCommitterContext {
     this.vertexName = vertexName;
     this.output = output;
     this.vertexIdx = vertexIdx;
+    this.dagIdentifier = dagIdentifier;
   }
 
   @Override
@@ -94,4 +97,9 @@ public class OutputCommitterContextImpl implements OutputCommitterContext {
     return vertexIdx;
   }
 
+  @Override
+  public int getDagIdentifier() {
+    return dagIdentifier;
+  }
+
 }
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index f8f275026..c7cf176af 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -2560,8 +2560,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl
                     appContext.getApplicationAttemptId().getAttemptId(),
                     appContext.getCurrentDAG().getName(),
                     vertexName,
-                    od,
-                    vertexId.getId());
+                    appContext.getCurrentDAG().getID().getId(),
+                    vertexId.getId(),
+                    od
+                );
             OutputCommitter outputCommitter = ReflectionUtils
                 .createClazzInstance(od.getControllerDescriptor().getClassName(),
                     new Class[]{OutputCommitterContext.class},
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java
index 71e5681cb..4a648dc90 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java
@@ -18,6 +18,7 @@
 
 package org.apache.tez.mapreduce.committer;
 
+import org.apache.tez.mapreduce.common.Utils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -78,6 +79,7 @@ public class MROutputCommitter extends OutputCommitter {
     jobConf.getCredentials().mergeAll(UserGroupInformation.getCurrentUser().getCredentials());
     jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
         getContext().getDAGAttemptNumber());
+    jobConf.set(MRJobConfig.JOB_COMMITTER_UUID, Utils.getDAGID(getContext()));
     jobConf.setInt(MRJobConfig.VERTEX_ID, getContext().getVertexIndex());
     committer = getOutputCommitter(getContext());
     jobContext = getJobContextFromVertexContext(getContext());
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/Utils.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/Utils.java
index 670ee5db4..85483fc59 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/Utils.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/Utils.java
@@ -30,7 +30,10 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FileSystem.Statistics;
 import org.apache.hadoop.mapred.Counters.Counter;
 import org.apache.tez.common.counters.TezCounter;
+import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.mapreduce.hadoop.mapred.MRCounters;
+import org.apache.tez.runtime.api.OutputCommitterContext;
+import org.apache.tez.runtime.api.OutputContext;
 
 @Private
 public final class Utils {
@@ -63,5 +66,12 @@ public final class Utils {
     Objects.requireNonNull(tezCounter);
     return new MRCounters.MRCounter(tezCounter);
   }
-  
+
+  public static String getDAGID(OutputCommitterContext context) {
+    return TezDAGID.getInstance(context.getApplicationId(), context.getDagIdentifier()).toString();
+  }
+
+  public static String getDAGID(OutputContext context) {
+    return TezDAGID.getInstance(context.getApplicationId(), context.getDagIdentifier()).toString();
+  }
 }
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
index e16246077..f1183742f 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
@@ -131,6 +131,11 @@ public interface MRJobConfig {
 
   public static final String CACHE_ARCHIVES_VISIBILITIES = "mapreduce.job.cache.archives.visibilities";
 
+  /**
+   * Used by committers to set a job-wide UUID.
+   */
+  public static final String JOB_COMMITTER_UUID = "job.committer.uuid";
+
   public static final String FILEOUTPUTCOMMITTER_ALGORITHM_VERSION = "mapreduce.fileoutputcommitter.algorithm.version";
 
   /**
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
index 9aeae25bd..b8ac1b3a5 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
@@ -33,6 +33,7 @@ import com.google.protobuf.ByteString;
 import org.apache.tez.common.Preconditions;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;
+import org.apache.tez.mapreduce.common.Utils;
 import org.apache.tez.runtime.library.api.IOInterruptedException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -413,6 +414,7 @@ public class MROutput extends AbstractLogicalOutput {
     }
     jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
         getContext().getDAGAttemptNumber());
+    jobConf.set(MRJobConfig.JOB_COMMITTER_UUID, Utils.getDAGID(getContext()));
     TaskAttemptID taskAttemptId = org.apache.tez.mapreduce.hadoop.mapreduce.TaskAttemptContextImpl
         .createMockTaskAttemptID(getContext().getApplicationId().getClusterTimestamp(),
             getContext().getTaskVertexIndex(), getContext().getApplicationId().getId(),
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutput.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutput.java
index bfc09dc9b..3359a6eda 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutput.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutput.java
@@ -56,7 +56,9 @@ import org.apache.tez.dag.api.UserPayload;
 import org.apache.tez.hadoop.shim.DefaultHadoopShim;
 import org.apache.tez.mapreduce.TestUmbilical;
 import org.apache.tez.mapreduce.TezTestUtils;
+import org.apache.tez.mapreduce.common.Utils;
 import org.apache.tez.mapreduce.hadoop.MRConfig;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 import org.apache.tez.runtime.LogicalIOProcessorRuntimeTask;
 import org.apache.tez.runtime.api.OutputContext;
 import org.apache.tez.runtime.api.ProcessorContext;
@@ -131,6 +133,26 @@ public class TestMROutput {
     assertEquals("base-value", mergedConf.get("base-key"));
   }
 
+  @Test
+  public void testJobUUIDSet() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setBoolean(MRConfig.IS_MAP_PROCESSOR, true);
+    DataSinkDescriptor dataSink = MROutput
+            .createConfigBuilder(conf, TextOutputFormat.class,
+                    tmpDir.getPath())
+            .build();
+
+    OutputContext outputContext = createMockOutputContext(dataSink.getOutputDescriptor().getUserPayload(),
+            new Configuration(false));
+    MROutput output = new MROutput(outputContext, 2);
+    output.initialize();
+    String invalidDAGID = "invalid default";
+    String dagID = output.jobConf.get(MRJobConfig.JOB_COMMITTER_UUID, invalidDAGID);
+    assertNotEquals(dagID, invalidDAGID);
+    assertNotEquals(output.jobConf.get(org.apache.hadoop.mapred.JobContext.TASK_ATTEMPT_ID), dagID);
+    assertEquals(dagID, Utils.getDAGID(outputContext));
+  }
+
   @Test(timeout = 5000)
   public void testOldAPI_TextOutputFormat() throws Exception {
     Configuration conf = new Configuration();