You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by pa...@apache.org on 2025/08/04 17:04:35 UTC

(hbase) branch branch-2 updated: HBASE-29477 Add configuration support for custom OutputCommitter in TableOutputFormat (#7185)

This is an automated email from the ASF dual-hosted git repository.

pankajkumar pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
     new c26953584d8 HBASE-29477 Add configuration support for custom OutputCommitter in TableOutputFormat (#7185)
c26953584d8 is described below

commit c26953584d8fd316e5f1385a4b82c7d645cff8a1
Author: Prathyush <48...@users.noreply.github.com>
AuthorDate: Mon Aug 4 22:34:29 2025 +0530

    HBASE-29477 Add configuration support for custom OutputCommitter in TableOutputFormat (#7185)
    
    Signed-off-by: Viraj Jasani <vj...@apache.org>
    Signed-off-by: Pankaj Kumar <pa...@apache.org>
    Reviewed-by: Ujjawal <uj...@gmail.com>
    Reviewed-by: Kevin Geiszler <ke...@gmail.com>
---
 .../hadoop/hbase/mapreduce/TableOutputFormat.java  | 24 ++++++++++++-
 .../hbase/mapreduce/TestTableOutputFormat.java     | 41 ++++++++++++++++++++++
 2 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
index 109aeef4ce0..54a357f5bef 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.ReflectionUtils;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.OutputCommitter;
 import org.apache.hadoop.mapreduce.OutputFormat;
@@ -72,6 +73,16 @@ public class TableOutputFormat<KEY> extends OutputFormat<KEY, Mutation> implemen
    */
   public static final String OUTPUT_CONF_PREFIX = "hbase.mapred.output.";
 
+  /**
+   * The configuration key for specifying a custom
+   * {@link org.apache.hadoop.mapreduce.OutputCommitter} implementation to be used by
+   * {@link TableOutputFormat}. The value for this property should be the fully qualified class name
+   * of the custom committer. If this property is not set, {@link TableOutputCommitter} will be used
+   * by default.
+   */
+  public static final String OUTPUT_COMMITTER_CLASS =
+    "hbase.mapreduce.tableoutputformat.output.committer.class";
+
   /**
    * Optional job parameter to specify a peer cluster. Used specifying remote cluster when copying
    * between hbase clusters (the source is picked up from <code>hbase-site.xml</code>).
@@ -216,7 +227,18 @@ public class TableOutputFormat<KEY> extends OutputFormat<KEY, Mutation> implemen
   @Override
   public OutputCommitter getOutputCommitter(TaskAttemptContext context)
     throws IOException, InterruptedException {
-    return new TableOutputCommitter();
+    Configuration hConf = getConf();
+    if (hConf == null) {
+      hConf = context.getConfiguration();
+    }
+
+    try {
+      Class<? extends OutputCommitter> outputCommitter =
+        hConf.getClass(OUTPUT_COMMITTER_CLASS, TableOutputCommitter.class, OutputCommitter.class);
+      return ReflectionUtils.newInstance(outputCommitter);
+    } catch (Exception e) {
+      throw new IOException("Could not create the configured OutputCommitter", e);
+    }
   }
 
   @Override
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableOutputFormat.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableOutputFormat.java
index 801099819b7..9170b26564d 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableOutputFormat.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableOutputFormat.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
 import org.apache.hadoop.mapreduce.RecordWriter;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.junit.After;
@@ -127,4 +129,43 @@ public class TestTableOutputFormat {
     Assert.assertEquals("Durability of the mutation should be SKIP_WAL", Durability.SKIP_WAL,
       delete.getDurability());
   }
+
+  @Test
+  public void testOutputCommitterConfiguration() throws IOException, InterruptedException {
+    // 1. Verify it returns the default committer when the property is not set.
+    conf.unset(TableOutputFormat.OUTPUT_COMMITTER_CLASS);
+    tableOutputFormat.setConf(conf);
+    Assert.assertEquals("Should use default committer", TableOutputCommitter.class,
+      tableOutputFormat.getOutputCommitter(context).getClass());
+
+    // 2. Verify it returns the custom committer when the property is set.
+    conf.set(TableOutputFormat.OUTPUT_COMMITTER_CLASS, DummyCommitter.class.getName());
+    tableOutputFormat.setConf(conf);
+    Assert.assertEquals("Should use custom committer", DummyCommitter.class,
+      tableOutputFormat.getOutputCommitter(context).getClass());
+  }
+
+  // Simple dummy committer for testing
+  public static class DummyCommitter extends OutputCommitter {
+    @Override
+    public void setupJob(JobContext jobContext) {
+    }
+
+    @Override
+    public void setupTask(TaskAttemptContext taskContext) {
+    }
+
+    @Override
+    public boolean needsTaskCommit(TaskAttemptContext taskContext) {
+      return false;
+    }
+
+    @Override
+    public void commitTask(TaskAttemptContext taskContext) {
+    }
+
+    @Override
+    public void abortTask(TaskAttemptContext taskContext) {
+    }
+  }
 }