[incubator-hivemall] branch master updated: [HIVEMALL-294] Fix XGboost to report progress report for each iteration

[my...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

myui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git


The following commit(s) were added to refs/heads/master by this push:
     new c356ecc  [HIVEMALL-294] Fix XGboost to report progress report for each iteration
c356ecc is described below

commit c356ecca7aaa8a2639b770a99faa0b8849edde61
Author: Makoto Yui <my...@apache.org>
AuthorDate: Fri May 29 16:42:14 2020 +0900

    [HIVEMALL-294] Fix XGboost to report progress report for each iteration
    
    ## What changes were proposed in this pull request?
    
    Fix XGboost to report progress report for each iteration.
    
    ## What type of PR is it?
    
    Improvement, Hot Fix
    
    ## What is the Jira issue?
    
    https://issues.apache.org/jira/browse/HIVEMALL-294
    
    ## How was this patch tested?
    
    unit tests
    
    ## Checklist
    
    - [x] Did you apply source code formatter, i.e., `./bin/format_code.sh`, for your commit?
    - [ ] Did you run system tests on Hive (or Spark)?
    
    Author: Makoto Yui <my...@apache.org>
    
    Closes #228 from myui/HIVEMALL-294.
---
 .../java/hivemall/xgboost/XGBoostTrainUDTF.java    | 25 ++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/xgboost/src/main/java/hivemall/xgboost/XGBoostTrainUDTF.java b/xgboost/src/main/java/hivemall/xgboost/XGBoostTrainUDTF.java
index 6922aca..4e0d1b8 100644
--- a/xgboost/src/main/java/hivemall/xgboost/XGBoostTrainUDTF.java
+++ b/xgboost/src/main/java/hivemall/xgboost/XGBoostTrainUDTF.java
@@ -46,6 +46,7 @@ import java.util.Random;
 
 import javax.annotation.Nonnegative;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Options;
@@ -62,6 +63,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapred.Reporter;
 
 /**
  * UDTF for train_xgboost
@@ -509,6 +512,8 @@ public class XGBoostTrainUDTF extends UDTFWithOptions {
 
     @Override
     public void close() throws HiveException {
+        final Reporter reporter = getReporter();
+
         DMatrix dmatrix = null;
         Booster booster = null;
         try {
@@ -531,13 +536,13 @@ public class XGBoostTrainUDTF extends UDTFWithOptions {
                 try {
                     dtest = dmatrix.slice(Arrays.copyOf(rows, numTest));
                     dtrain = dmatrix.slice(Arrays.copyOfRange(rows, numTest, rows.length));
-                    booster = train(dtrain, dtest, round, earlyStoppingRounds, params);
+                    booster = train(dtrain, dtest, round, earlyStoppingRounds, params, reporter);
                 } finally {
                     XGBoostUtils.close(dtrain);
                     XGBoostUtils.close(dtest);
                 }
             } else {
-                booster = train(dmatrix, round, params);
+                booster = train(dmatrix, round, params, reporter);
             }
             onFinishTraining(booster);
 
@@ -560,11 +565,17 @@ public class XGBoostTrainUDTF extends UDTFWithOptions {
 
     @Nonnull
     private static Booster train(@Nonnull final DMatrix dtrain, @Nonnegative final int round,
-            @Nonnull final Map<String, Object> params)
+            @Nonnull final Map<String, Object> params, @Nullable final Reporter reporter)
             throws NoSuchMethodException, IllegalAccessException, InvocationTargetException,
             InstantiationException, XGBoostError {
+        final Counters.Counter iterCounter = (reporter == null) ? null
+                : reporter.getCounter("hivemall.XGBoostTrainUDTF$Counter", "iteration");
+
         final Booster booster = XGBoostUtils.createBooster(dtrain, params);
         for (int iter = 0; iter < round; iter++) {
+            reportProgress(reporter);
+            setCounterValue(iterCounter, iter + 1);
+
             booster.update(dtrain, iter);
         }
         return booster;
@@ -573,9 +584,12 @@ public class XGBoostTrainUDTF extends UDTFWithOptions {
     @Nonnull
     private static Booster train(@Nonnull final DMatrix dtrain, @Nonnull final DMatrix dtest,
             @Nonnegative final int round, @Nonnegative final int earlyStoppingRounds,
-            @Nonnull final Map<String, Object> params)
+            @Nonnull final Map<String, Object> params, @Nullable final Reporter reporter)
             throws NoSuchMethodException, IllegalAccessException, InvocationTargetException,
             InstantiationException, XGBoostError {
+        final Counters.Counter iterCounter = (reporter == null) ? null
+                : reporter.getCounter("hivemall.XGBoostTrainUDTF$Counter", "iteration");
+
         final Booster booster = XGBoostUtils.createBooster(dtrain, params);
 
         final boolean maximizeEvaluationMetrics =
@@ -585,6 +599,9 @@ public class XGBoostTrainUDTF extends UDTFWithOptions {
 
         final float[] metricsOut = new float[1];
         for (int iter = 0; iter < round; iter++) {
+            reportProgress(reporter);
+            setCounterValue(iterCounter, iter + 1);
+
             booster.update(dtrain, iter);
 
             String evalInfo =