You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2019/11/22 15:56:43 UTC
[incubator-hivemall] branch master updated: [HIVEMALL-276] Stable
support for XGBoost v0.90
This is an automated email from the ASF dual-hosted git repository.
myui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git
The following commit(s) were added to refs/heads/master by this push:
new 54e1d7d [HIVEMALL-276] Stable support for XGBoost v0.90
54e1d7d is described below
commit 54e1d7da67690b17640809e60c7b251fdc82308a
Author: Makoto Yui <my...@apache.org>
AuthorDate: Sat Nov 23 00:56:36 2019 +0900
[HIVEMALL-276] Stable support for XGBoost v0.90
## What changes were proposed in this pull request?
- Fix xgboost module to create DMatrix from CSRMatrix
- Support xgboost v0.90 hyperparameters
- Replace xgboost4j with [xgboost-predictor](https://github.com/komiya-atsushi/xgboost-predictor-java) for prediction
- Add documentation about Xgboost
## What type of PR is it?
Refactoring, Improvement
## What is the Jira issue?
https://issues.apache.org/jira/browse/HIVEMALL-276
https://issues.apache.org/jira/browse/HIVEMALL-275
https://issues.apache.org/jira/browse/HIVEMALL-279
https://issues.apache.org/jira/browse/HIVEMALL-272
https://issues.apache.org/jira/browse/HIVEMALL-27
## How to use this feature?
as described in [user guide](http://hivemall.apache.org/userguide/index.html).
## How was this patch tested?
unit tests and manual tests on EMR
## Checklist
- [x] Did you apply source code formatter, i.e., `./bin/format_code.sh`, for your commit?
- [x] Did you run system tests on Hive (or Spark)?
Author: Makoto Yui <my...@apache.org>
Closes #213 from myui/HIVEMALL-275-2.
---
core/src/main/java/hivemall/UDTFWithOptions.java | 27 +
.../hivemall/classifier/GeneralClassifierUDTF.java | 2 +-
.../hivemall/tools/array/ArrayAvgGenericUDAF.java | 10 +-
.../main/java/hivemall/utils/hadoop/HiveUtils.java | 96 +++-
.../java/hivemall/utils/hadoop/WritableUtils.java | 47 ++
core/src/main/java/hivemall/utils/io/IOUtils.java | 46 ++
.../main/java/hivemall/utils/lang/ArrayUtils.java | 24 +-
.../main/java/hivemall/utils/lang/ObjectUtils.java | 3 +-
.../main/java/hivemall/utils/lang/OptionUtils.java | 83 +++
.../test/java/hivemall/utils/io/IOUtilsTest.java | 36 +-
dist/pom.xml | 1 +
docs/gitbook/SUMMARY.md | 4 +
docs/gitbook/binaryclass/news20b_xgboost.md | 413 ++++++++++++++
docs/gitbook/misc/funcs.md | 75 ++-
docs/gitbook/multiclass/iris_xgboost.md | 146 +++++
docs/gitbook/multiclass/news20_dataset.md | 7 +-
docs/gitbook/multiclass/news20_pa.md | 30 +-
docs/gitbook/multiclass/news20_scw.md | 76 +--
docs/gitbook/multiclass/news20_xgboost.md | 137 +++++
docs/gitbook/regression/e2006_xgboost.md | 107 ++++
pom.xml | 4 +
resources/ddl/define-additional.hive | 25 +-
resources/ddl/define-all-as-permanent.hive | 24 +-
.../java/hivemall/docs/FuncsListGeneratorMojo.java | 3 +-
xgboost/pom.xml | 12 +-
.../java/hivemall/xgboost/XGBoostBaseUDTF.java | 337 -----------
.../hivemall/xgboost/XGBoostBatchPredictUDTF.java | 350 ++++++++++++
.../hivemall/xgboost/XGBoostOnlinePredictUDTF.java | 252 +++++++++
.../hivemall/xgboost/XGBoostPredictOneUDTF.java | 79 +++
.../hivemall/xgboost/XGBoostPredictTripleUDTF.java | 86 +++
.../java/hivemall/xgboost/XGBoostPredictUDTF.java | 225 --------
.../java/hivemall/xgboost/XGBoostTrainUDTF.java | 629 +++++++++++++++++++++
.../main/java/hivemall/xgboost/XGBoostUtils.java | 76 ---
.../java/hivemall/xgboost/XGBoostVersionUDF.java | 2 +
.../XGBoostBinaryClassifierUDTF.java | 52 --
.../XGBoostMulticlassClassifierUDTF.java | 82 ---
.../xgboost/regression/XGBoostRegressionUDTF.java | 51 --
.../tools/XGBoostMulticlassPredictUDTF.java | 80 ---
.../hivemall/xgboost/tools/XGBoostPredictUDTF.java | 75 ---
.../hivemall/xgboost/utils/DMatrixBuilder.java | 109 ++++
.../xgboost/utils/DenseDMatrixBuilder.java | 93 +++
.../xgboost/{ => utils}/NativeLibLoader.java | 2 +-
.../xgboost/utils/SparseDMatrixBuilder.java | 66 ++-
.../java/hivemall/xgboost/utils/XGBoostUtils.java | 165 ++++++
xgboost/src/test/java/hivemall/TestBase.java | 450 +++++++++++++++
xgboost/src/test/java/hivemall/TestUtils.java | 126 +++++
.../hivemall/xgboost/XGBoostTrainUDTFTest.java | 288 ++++++++++
.../hivemall/xgboost/utils/DMatrixBuilderTest.java | 103 ++++
48 files changed, 4073 insertions(+), 1143 deletions(-)
diff --git a/core/src/main/java/hivemall/UDTFWithOptions.java b/core/src/main/java/hivemall/UDTFWithOptions.java
index a71395e..853785b 100644
--- a/core/src/main/java/hivemall/UDTFWithOptions.java
+++ b/core/src/main/java/hivemall/UDTFWithOptions.java
@@ -28,6 +28,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnegative;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -171,6 +173,31 @@ public abstract class UDTFWithOptions extends GenericUDTF {
return list;
}
+ protected static <T> T checkNotNull(@CheckForNull final T arg, @Nonnegative final String errMsg)
+ throws UDFArgumentException {
+ if (arg == null) {
+ throw new UDFArgumentException(errMsg);
+ }
+ return arg;
+ }
+
+ protected static <T> T checkNotNull(@CheckForNull final T arg, @Nonnegative final int index)
+ throws UDFArgumentException {
+ if (arg == null) {
+ throw new UDFArgumentException(String.format("%d-th argument MUST not be null", index));
+ }
+ return arg;
+ }
+
+ protected static Object nonNullArgument(@Nonnull final Object[] args,
+ @Nonnegative final int index) throws UDFArgumentException {
+ final Object arg = args[index];
+ if (arg == null) {
+ throw new UDFArgumentException(String.format("%d-th argument MUST not be null", index));
+ }
+ return arg;
+ }
+
/**
* Raise {@link UDFArgumentException} if the given condition is false.
*
diff --git a/core/src/main/java/hivemall/classifier/GeneralClassifierUDTF.java b/core/src/main/java/hivemall/classifier/GeneralClassifierUDTF.java
index 98cdf0b..6213859 100644
--- a/core/src/main/java/hivemall/classifier/GeneralClassifierUDTF.java
+++ b/core/src/main/java/hivemall/classifier/GeneralClassifierUDTF.java
@@ -60,7 +60,7 @@ public final class GeneralClassifierUDTF extends GeneralLearnerBaseUDTF {
@Override
protected void checkTargetValue(final float label) throws UDFArgumentException {
if (label != -1 && label != 0 && label != 1) {
- throw new UDFArgumentException("Invalid label value for classification: + label");
+ throw new UDFArgumentException("Invalid label value for classification: " + label);
}
}
diff --git a/core/src/main/java/hivemall/tools/array/ArrayAvgGenericUDAF.java b/core/src/main/java/hivemall/tools/array/ArrayAvgGenericUDAF.java
index ad9c39a..d9e369c 100644
--- a/core/src/main/java/hivemall/tools/array/ArrayAvgGenericUDAF.java
+++ b/core/src/main/java/hivemall/tools/array/ArrayAvgGenericUDAF.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AbstractAggregationBuffer;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationType;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryArray;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -55,7 +56,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObj
import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableIntObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableLongObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
// @formatter:off
@@ -130,7 +130,7 @@ public final class ArrayAvgGenericUDAF extends AbstractGenericUDAFResolver {
outputOI = internalMergeOI();
} else {// terminate
outputOI = ObjectInspectorFactory.getStandardListObjectInspector(
- PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
+ PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
}
return outputOI;
}
@@ -222,7 +222,7 @@ public final class ArrayAvgGenericUDAF extends AbstractGenericUDAFResolver {
}
@Override
- public List<FloatWritable> terminate(
+ public List<DoubleWritable> terminate(
@SuppressWarnings("deprecation") AggregationBuffer aggr) throws HiveException {
ArrayAvgAggregationBuffer myAggr = (ArrayAvgAggregationBuffer) aggr;
@@ -234,11 +234,11 @@ public final class ArrayAvgGenericUDAF extends AbstractGenericUDAFResolver {
final double[] sum = myAggr._sum;
final long[] count = myAggr._count;
- final FloatWritable[] ary = new FloatWritable[size];
+ final DoubleWritable[] ary = new DoubleWritable[size];
for (int i = 0; i < size; i++) {
long c = count[i];
float avg = (c == 0) ? 0.f : (float) (sum[i] / c);
- ary[i] = new FloatWritable(avg);
+ ary[i] = new DoubleWritable(avg);
}
return Arrays.asList(ary);
}
diff --git a/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java b/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java
index 9379c1e..5c485cd 100644
--- a/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java
+++ b/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java
@@ -115,6 +115,17 @@ public final class HiveUtils {
return Integer.parseInt(s);
}
+ @Nullable
+ public static Writable copyToWritable(@Nullable final Object o,
+ @Nonnull final PrimitiveObjectInspector poi) {
+ if (o == null) {
+ return null;
+ }
+ Object copied = poi.copyObject(o);
+ Object result = poi.getPrimitiveWritableObject(copied);
+ return (Writable) result;
+ }
+
public static Text asText(@Nullable final Object o) {
if (o == null) {
return null;
@@ -562,6 +573,7 @@ public final class HiveUtils {
return ary;
}
+ @Nullable
public static String getConstString(@Nonnull final ObjectInspector oi)
throws UDFArgumentException {
if (!isStringOI(oi)) {
@@ -572,6 +584,19 @@ public final class HiveUtils {
return v == null ? null : v.toString();
}
+
+ @Nullable
+ public static String getConstString(@Nonnull final ObjectInspector[] argOIs, final int argIndex)
+ throws UDFArgumentException {
+ final ObjectInspector oi = getObjectInspector(argOIs, argIndex);
+ if (!isStringOI(oi)) {
+ throw new UDFArgumentException("argOIs[" + argIndex + "] must be a Text value: "
+ + TypeInfoUtils.getTypeInfoFromObjectInspector(oi));
+ }
+ Text v = getConstValue(oi);
+ return v == null ? null : v.toString();
+ }
+
public static boolean getConstBoolean(@Nonnull final ObjectInspector oi)
throws UDFArgumentException {
if (!isBooleanOI(oi)) {
@@ -882,6 +907,14 @@ public final class HiveUtils {
return (ConstantObjectInspector) oi;
}
+ public static ObjectInspector getObjectInspector(@Nonnull final ObjectInspector[] argOIs,
+ final int argIndex) throws UDFArgumentException {
+ if (argIndex >= argOIs.length) {
+ throw new UDFArgumentException("Illegal argument index:" + argIndex);
+ }
+ return argOIs[argIndex];
+ }
+
@Nonnull
public static PrimitiveObjectInspector asPrimitiveObjectInspector(
@Nonnull final ObjectInspector oi) throws UDFArgumentException {
@@ -893,15 +926,38 @@ public final class HiveUtils {
}
@Nonnull
+ public static PrimitiveObjectInspector asPrimitiveObjectInspector(
+ @Nonnull final ObjectInspector[] argOIs, final int argIndex)
+ throws UDFArgumentException {
+ final ObjectInspector oi = getObjectInspector(argOIs, argIndex);
+ if (oi.getCategory() != Category.PRIMITIVE) {
+ throw new UDFArgumentException("Expecting PrimitiveObjectInspector for argOIs["
+ + argIndex + "] but got " + TypeInfoUtils.getTypeInfoFromObjectInspector(oi));
+ }
+ return (PrimitiveObjectInspector) oi;
+ }
+
+ @Nonnull
public static StringObjectInspector asStringOI(@Nonnull final ObjectInspector argOI)
throws UDFArgumentException {
- if (!STRING_TYPE_NAME.equals(argOI.getTypeName())) {
+ if (!isStringOI(argOI)) {
throw new UDFArgumentException("Argument type must be String: " + argOI.getTypeName());
}
return (StringObjectInspector) argOI;
}
@Nonnull
+ public static StringObjectInspector asStringOI(@Nonnull final ObjectInspector[] argOIs,
+ final int argIndex) throws UDFArgumentException {
+ final ObjectInspector oi = getObjectInspector(argOIs, argIndex);
+ if (!isStringOI(oi)) {
+ throw new UDFArgumentException(
+ "argOIs[" + argIndex + "] type must be String: " + oi.getTypeName());
+ }
+ return (StringObjectInspector) oi;
+ }
+
+ @Nonnull
public static BinaryObjectInspector asBinaryOI(@Nonnull final ObjectInspector argOI)
throws UDFArgumentException {
if (!BINARY_TYPE_NAME.equals(argOI.getTypeName())) {
@@ -1048,6 +1104,31 @@ public final class HiveUtils {
}
@Nonnull
+ public static PrimitiveObjectInspector asDoubleCompatibleOI(
+ @Nonnull final ObjectInspector[] argOIs, final int argIndex)
+ throws UDFArgumentException {
+ final PrimitiveObjectInspector oi = asPrimitiveObjectInspector(argOIs, argIndex);
+ switch (oi.getPrimitiveCategory()) {
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+ case FLOAT:
+ case DOUBLE:
+ case DECIMAL:
+ case STRING:
+ case TIMESTAMP:
+ break;
+ default:
+ throw new UDFArgumentTypeException(argIndex,
+ "Only numeric or string type arguments are accepted but " + oi.getTypeName()
+ + " is passed for argument index " + argIndex);
+ }
+ return oi;
+
+ }
+
+ @Nonnull
public static PrimitiveObjectInspector asFloatingPointOI(@Nonnull final ObjectInspector argOI)
throws UDFArgumentTypeException {
if (argOI.getCategory() != Category.PRIMITIVE) {
@@ -1102,6 +1183,19 @@ public final class HiveUtils {
}
@Nonnull
+ public static ListObjectInspector asListOI(@Nonnull final ObjectInspector[] argOIs,
+ final int argIndex) throws UDFArgumentException {
+ final ObjectInspector oi = getObjectInspector(argOIs, argIndex);
+ Category category = oi.getCategory();
+ if (category != Category.LIST) {
+ throw new UDFArgumentException("Expecting ListObjectInspector for argOIs[" + argIndex
+ + "] but got " + TypeInfoUtils.getTypeInfoFromObjectInspector(oi));
+ }
+ return (ListObjectInspector) oi;
+ }
+
+
+ @Nonnull
public static MapObjectInspector asMapOI(@Nonnull final ObjectInspector oi)
throws UDFArgumentException {
if (oi.getCategory() != Category.MAP) {
diff --git a/core/src/main/java/hivemall/utils/hadoop/WritableUtils.java b/core/src/main/java/hivemall/utils/hadoop/WritableUtils.java
index 280107d..a364d74 100644
--- a/core/src/main/java/hivemall/utils/hadoop/WritableUtils.java
+++ b/core/src/main/java/hivemall/utils/hadoop/WritableUtils.java
@@ -91,6 +91,30 @@ public final class WritableUtils {
}
@Nonnull
+ public static List<FloatWritable> newFloatList(final int size) {
+ return newFloatList(size, 0.f);
+ }
+
+ @Nonnull
+ public static List<FloatWritable> newFloatList(final int size, final float defaultValue) {
+ // workaround to avoid a bug in Kryo
+ // https://issues.apache.org/jira/browse/HIVE-12551
+ /*
+ final DoubleWritable[] array = new DoubleWritable[size];
+ for (int i = 0; i < size; i++) {
+ array[i] = new DoubleWritable(defaultValue);
+ }
+ return Arrays.asList(array);
+ */
+ final List<FloatWritable> list = new ArrayList<FloatWritable>(size);
+ for (int i = 0; i < size; i++) {
+ list.add(new FloatWritable(defaultValue));
+ }
+ return list;
+ }
+
+
+ @Nonnull
public static List<DoubleWritable> newDoubleList(final int size, final double defaultValue) {
// workaround to avoid a bug in Kryo
// https://issues.apache.org/jira/browse/HIVE-12551
@@ -172,6 +196,20 @@ public final class WritableUtils {
}
@Nonnull
+ public static List<FloatWritable> toWritableList(@Nonnull final float[] src,
+ @Nullable List<FloatWritable> list) throws UDFArgumentException {
+ if (list == null) {
+ return toWritableList(src);
+ }
+
+ Preconditions.checkArgument(src.length == list.size(), UDFArgumentException.class);
+ for (int i = 0; i < src.length; i++) {
+ list.set(i, new FloatWritable(src[i]));
+ }
+ return list;
+ }
+
+ @Nonnull
public static List<DoubleWritable> toWritableList(@Nonnull final double[] src,
@Nullable List<DoubleWritable> list) throws UDFArgumentException {
if (list == null) {
@@ -185,6 +223,15 @@ public final class WritableUtils {
return list;
}
+ @Nonnull
+ public static void setValues(@Nonnull final float[] src,
+ @Nonnull final List<FloatWritable> list) throws UDFArgumentException {
+ Preconditions.checkArgument(src.length == list.size(), UDFArgumentException.class);
+ for (int i = 0; i < src.length; i++) {
+ list.set(i, new FloatWritable(src[i]));
+ }
+ }
+
public static Text val(final String v) {
return new Text(v);
}
diff --git a/core/src/main/java/hivemall/utils/io/IOUtils.java b/core/src/main/java/hivemall/utils/io/IOUtils.java
index 27d4b49..a6f8e9b 100644
--- a/core/src/main/java/hivemall/utils/io/IOUtils.java
+++ b/core/src/main/java/hivemall/utils/io/IOUtils.java
@@ -19,6 +19,7 @@
package hivemall.utils.io;
import hivemall.utils.codec.ZigZagLEB128Codec;
+import hivemall.utils.io.CompressionStreamFactory.CompressionAlgorithm;
import java.io.BufferedReader;
import java.io.Closeable;
@@ -351,4 +352,49 @@ public final class IOUtils {
readFully(in, b, 0, b.length);
}
+ @Nonnull
+ public static byte[] toCompressedText(@Nonnull final byte[] in) throws IOException {
+ return toCompressedText(in, in.length);
+ }
+
+ @Nonnull
+ public static byte[] toCompressedText(@Nonnull final byte[] in, final int len)
+ throws IOException {
+ final FastByteArrayInputStream fis = new FastByteArrayInputStream(in, len);
+ final FastMultiByteArrayOutputStream fos = new FastMultiByteArrayOutputStream();
+
+ FinishableOutputStream dos = null;
+ try {
+ Base91OutputStream bos = new Base91OutputStream(fos);
+ dos = CompressionStreamFactory.createOutputStream(bos, CompressionAlgorithm.deflate);
+ copy(fis, dos);
+ dos.finish(); // flush is called
+ return fos.toByteArray_clear();
+ } finally {
+ IOUtils.closeQuietly(dos);
+ }
+ }
+
+ @Nonnull
+ public static byte[] fromCompressedText(@Nonnull final byte[] src) throws IOException {
+ return fromCompressedText(src, src.length);
+ }
+
+ @Nonnull
+ public static byte[] fromCompressedText(@Nonnull final byte[] src, final int len)
+ throws IOException {
+ final FastByteArrayInputStream bis = new FastByteArrayInputStream(src, len);
+ final FastMultiByteArrayOutputStream bos = new FastMultiByteArrayOutputStream();
+
+ InputStream compressedStream = null;
+ try {
+ compressedStream = CompressionStreamFactory.createInputStream(
+ new Base91InputStream(bis), CompressionAlgorithm.deflate);
+ copy(compressedStream, bos);
+ return bos.toByteArray_clear();
+ } finally {
+ IOUtils.closeQuietly(compressedStream);
+ }
+ }
+
}
diff --git a/core/src/main/java/hivemall/utils/lang/ArrayUtils.java b/core/src/main/java/hivemall/utils/lang/ArrayUtils.java
index bb4ce22..5dbb834 100644
--- a/core/src/main/java/hivemall/utils/lang/ArrayUtils.java
+++ b/core/src/main/java/hivemall/utils/lang/ArrayUtils.java
@@ -1000,7 +1000,7 @@ public final class ArrayUtils {
public static int argmin(@Nonnull final double[] a) {
final int size = a.length;
if (size == 0) {
- throw new IllegalArgumentException("attempt to get argmax of an empty array");
+ throw new IllegalArgumentException("attempt to get argmin of an empty array");
}
int minIdx = 0;
@@ -1018,7 +1018,7 @@ public final class ArrayUtils {
public static <T extends Comparable<T>> int argmin(@Nonnull final T[] a) {
final int size = a.length;
if (size == 0) {
- throw new IllegalArgumentException("attempt to get argmax of an empty array");
+ throw new IllegalArgumentException("attempt to get argmin of an empty array");
}
int minIdx = 0;
@@ -1040,7 +1040,7 @@ public final class ArrayUtils {
public static <T> int argmin(@Nonnull final T[] a, @Nonnull final Comparator<? super T> c) {
final int size = a.length;
if (size == 0) {
- throw new IllegalArgumentException("attempt to get argmax of an empty array");
+ throw new IllegalArgumentException("attempt to get argmin of an empty array");
}
if (size == 1) {
return 0;
@@ -1058,6 +1058,24 @@ public final class ArrayUtils {
return minIdx;
}
+ public static int argmax(@Nonnull final double[] a) {
+ final int size = a.length;
+ if (size == 0) {
+ throw new IllegalArgumentException("attempt to get argmin of an empty array");
+ }
+
+ int maxIdx = 0;
+ double maxVaue = Double.NEGATIVE_INFINITY;
+ for (int i = 0; i < size; i++) {
+ final double v = a[i];
+ if (v > maxVaue) {
+ maxIdx = i;
+ maxVaue = v;
+ }
+ }
+ return maxIdx;
+ }
+
public static <T extends Comparable<T>> int argmax(@Nonnull final T[] a) {
final int size = a.length;
if (size == 0) {
diff --git a/core/src/main/java/hivemall/utils/lang/ObjectUtils.java b/core/src/main/java/hivemall/utils/lang/ObjectUtils.java
index 459f135..8e122ec 100644
--- a/core/src/main/java/hivemall/utils/lang/ObjectUtils.java
+++ b/core/src/main/java/hivemall/utils/lang/ObjectUtils.java
@@ -90,8 +90,7 @@ public final class ObjectUtils {
out = bin2txt ? new Base91OutputStream(bos) : bos;
dos = CompressionStreamFactory.createOutputStream(out, algo);
toStream(obj, dos);
- dos.finish();
- //dos.flush();
+ dos.finish(); // flush is called
return bos.toByteArray_clear();
} finally {
IOUtils.closeQuietly(dos);
diff --git a/core/src/main/java/hivemall/utils/lang/OptionUtils.java b/core/src/main/java/hivemall/utils/lang/OptionUtils.java
new file mode 100644
index 0000000..d76aed6
--- /dev/null
+++ b/core/src/main/java/hivemall/utils/lang/OptionUtils.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.utils.lang;
+
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import javax.annotation.Nonnull;
+
+public final class OptionUtils {
+
+ private OptionUtils() {}
+
+ public static boolean getBoolean(@Nonnull final Map<String, ?> options,
+ @Nonnull final String optionName) {
+ Object value = options.get(optionName);
+ if (value == null) {
+ throw new NoSuchElementException("Could not find an option: " + optionName);
+ }
+ if (!(value instanceof Boolean)) {
+ throw new IllegalStateException(
+ "Non-boolean unexpected value type: " + value.getClass().getSimpleName());
+ }
+ return ((Boolean) value).booleanValue();
+ }
+
+ public static int getInt(@Nonnull final Map<String, ?> options,
+ @Nonnull final String optionName) {
+ Object value = options.get(optionName);
+ if (value == null) {
+ throw new NoSuchElementException("Could not find an option: " + optionName);
+ }
+ if (!(value instanceof Integer)) {
+ throw new IllegalStateException(
+ "Non-integer unexpected value type: " + value.getClass().getSimpleName());
+ }
+ return ((Integer) value).intValue();
+ }
+
+ public static long getLong(@Nonnull final Map<String, ?> options,
+ @Nonnull final String optionName) {
+ Object value = options.get(optionName);
+ if (value == null) {
+ throw new NoSuchElementException("Could not find an option: " + optionName);
+ }
+ if (!(value instanceof Long)) {
+ throw new IllegalStateException(
+ "Non-long unexpected value type: " + value.getClass().getSimpleName());
+ }
+ return ((Long) value).longValue();
+ }
+
+ public static double getDouble(@Nonnull final Map<String, ?> options,
+ @Nonnull final String optionName) {
+ Object value = options.get(optionName);
+ if (value == null) {
+ throw new NoSuchElementException("Could not find an option: " + optionName);
+ }
+ if (!(value instanceof Double)) {
+ throw new IllegalStateException(
+ "Non-double unexpected value type: " + value.getClass().getSimpleName());
+ }
+ return ((Double) value).doubleValue();
+ }
+
+
+}
diff --git a/xgboost/src/main/java/hivemall/xgboost/XGBoostVersionUDF.java b/core/src/test/java/hivemall/utils/io/IOUtilsTest.java
similarity index 52%
copy from xgboost/src/main/java/hivemall/xgboost/XGBoostVersionUDF.java
copy to core/src/test/java/hivemall/utils/io/IOUtilsTest.java
index 0c08295..f4643f7 100644
--- a/xgboost/src/main/java/hivemall/xgboost/XGBoostVersionUDF.java
+++ b/core/src/test/java/hivemall/utils/io/IOUtilsTest.java
@@ -16,30 +16,26 @@
* specific language governing permissions and limitations
* under the License.
*/
-package hivemall.xgboost;
+package hivemall.utils.io;
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
+import java.io.IOException;
-import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.exec.UDF;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
-@Description(name = "xgboost_version", value = "_FUNC_() - Returns the version of xgboost",
- extended = "SELECT xgboost_version();")
-@UDFType(deterministic = true, stateful = false)
-public final class XGBoostVersionUDF extends UDF {
+public class IOUtilsTest {
- @Nullable
- private String version;
-
- @Nonnull
- public String evaluate() throws HiveException {
- if (version == null) {
- this.version = XGBoostUtils.getVersion();
- }
- return version;
+ @Test
+ public void testCompressedText() throws IOException, ClassNotFoundException {
+ Text in = new Text("Licensed to the Apache Software Foundation (ASF)");
+ byte[] expected = in.copyBytes();
+ byte[] compressed = IOUtils.toCompressedText(expected);
+ Text encoded = new Text(compressed);
+ byte[] actual = IOUtils.fromCompressedText(encoded.getBytes(), encoded.getLength());
+ Assert.assertArrayEquals(expected, actual);
+ Text out = new Text(actual);
+ Assert.assertEquals(in, out);
}
}
diff --git a/dist/pom.xml b/dist/pom.xml
index b544bee..1b0da97 100644
--- a/dist/pom.xml
+++ b/dist/pom.xml
@@ -125,6 +125,7 @@
<!-- hivemall-xgboost -->
<include>org.apache.hivemall:hivemall-xgboost</include>
<include>io.github.myui:xgboost4j</include>
+ <include>biz.k11i:xgboost-predictor</include>
<include>com.esotericsoftware.kryo:kryo</include>
<!-- for Jason encoding/decoding -->
<include>org.codehaus.jackson:jackson-core-asl</include>
diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md
index c50e78e..9e86cf7 100644
--- a/docs/gitbook/SUMMARY.md
+++ b/docs/gitbook/SUMMARY.md
@@ -102,6 +102,7 @@
* [General Binary Classifier](binaryclass/news20_generic.md)
* [AdaGradRDA, AdaGrad, AdaDelta](binaryclass/news20_adagrad.md)
* [Random Forest](binaryclass/news20_rf.md)
+ * [XGBoost](binaryclass/news20b_xgboost.md)
* [KDD2010a Tutorial](binaryclass/kdd2010a.md)
* [Data Preparation](binaryclass/kdd2010a_dataset.md)
@@ -128,6 +129,7 @@
* [Data Preparation for one-vs-the-rest classifiers](multiclass/news20_one-vs-the-rest_dataset.md)
* [PA](multiclass/news20_pa.md)
* [CW, AROW, SCW](multiclass/news20_scw.md)
+ * [XGBoost](multiclass/news20_xgboost.md)
* [Ensemble learning](multiclass/news20_ensemble.md)
* [one-vs-the-rest Classifier](multiclass/news20_one-vs-the-rest.md)
@@ -135,6 +137,7 @@
* [Data preparation](multiclass/iris_dataset.md)
* [SCW](multiclass/iris_scw.md)
* [Random Forest](multiclass/iris_randomforest.md)
+ * [XGBoost](multiclass/iris_xgboost.md)
## Part VIII - Regression
@@ -144,6 +147,7 @@
* [Data Preparation](regression/e2006_dataset.md)
* [General Regessor](regression/e2006_generic.md)
* [Passive Aggressive, AROW](regression/e2006_arow.md)
+ * [XGBoost](regression/e2006_xgboost.md)
* [KDDCup 2012 Track 2 CTR Prediction Tutorial](regression/kddcup12tr2.md)
* [Data Preparation](regression/kddcup12tr2_dataset.md)
diff --git a/docs/gitbook/binaryclass/news20b_xgboost.md b/docs/gitbook/binaryclass/news20b_xgboost.md
new file mode 100644
index 0000000..3dace0e
--- /dev/null
+++ b/docs/gitbook/binaryclass/news20b_xgboost.md
@@ -0,0 +1,413 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+In this tutorial, we build a binary classification model using XGBoost.
+
+<!-- toc -->
+
+## Feature Vector format for XGBoost
+
+For feature vector, `train_xgboost` takes a sparse vector format (`array<string>`) or a dense vector format (`array<double>`).
+In the feature vector, each feature takes a LIBSVM format:
+
+```
+feature ::= <index>:<weight>
+
+index ::= <Non-negative INT> (e.g., 0,1,2,...)
+weight ::= <DOUBLE>
+```
+
+> #### Note
+> Unlike the original libsvm format, it's not needed to sort a feature vector by ansceding order of feature index.
+
+Target label format of binary classification follows [this rule](http://hivemall.apache.org/userguide/getting_started/input-format.html#label-format-in-binary-classification). Please refer [xgboost document](https://xgboost.readthedocs.io/en/latest/tutorials/input_format.html) as well.
+
+## Label format in Binary Classification
+
+The label must be an INT typed column and the values are positive (+1) or negative (-1) as follows:
+
+```
+<label> ::= 1 | -1
+```
+
+Alternatively, you can use the following format that represents 1 for a positive example and 0 for a negative example:
+
+```
+<label> ::= 0 | 1
+```
+
+## Usage and Hyperparameters
+
+You can find hyperparameters and it's default setting by running the following query:
+
+```sql
+select train_xgboost();
+
+usage: train_xgboost(array<string|double> features, int|double target [,
+ string options]) - Returns a relation consists of <string model_id,
+ array<string> pred_model> [-alpha <arg>] [-base_score <arg>]
+ [-booster <arg>] [-colsample_bylevel <arg>] [-colsample_bynode
+ <arg>] [-colsample_bytree <arg>] [-disable_default_eval_metric
+ <arg>] [-eta <arg>] [-eval_metric <arg>] [-feature_selector <arg>]
+ [-gamma <arg>] [-grow_policy <arg>] [-lambda <arg>] [-lambda_bias
+ <arg>] [-max_bin <arg>] [-max_delta_step <arg>] [-max_depth <arg>]
+ [-max_leaves <arg>] [-maximize_evaluation_metrics <arg>]
+ [-min_child_weight <arg>] [-normalize_type <arg>] [-num_class
+ <arg>] [-num_early_stopping_rounds <arg>] [-num_feature <arg>]
+ [-num_parallel_tree <arg>] [-num_pbuffer <arg>] [-num_round <arg>]
+ [-objective <arg>] [-one_drop <arg>] [-process_type <arg>]
+ [-rate_drop <arg>] [-refresh_leaf <arg>] [-sample_type <arg>]
+ [-scale_pos_weight <arg>] [-seed <arg>] [-silent <arg>]
+ [-sketch_eps <arg>] [-skip_drop <arg>] [-subsample <arg>] [-top_k
+ <arg>] [-tree_method <arg>] [-tweedie_variance_power <arg>]
+ [-updater <arg>] [-validation_ratio <arg>] [-verbosity <arg>]
+ -alpha,--reg_alpha <arg> L1 regularization term on weights.
+ Increasing this value will make
+ model more conservative. [default:
+ 0.0]
+ -base_score <arg> Initial prediction score of all
+ instances, global bias [default:
+ 0.5]
+ -booster <arg> Set a booster to use, gbtree or
+ gblinear or dart. [default: gbree]
+ -colsample_bylevel <arg> Subsample ratio of columns for each
+ level [default: 1.0]
+ -colsample_bynode <arg> Subsample ratio of columns for each
+ node [default: 1.0]
+ -colsample_bytree <arg> Subsample ratio of columns when
+ constructing each tree [default:
+ 1.0]
+ -disable_default_eval_metric <arg> NFlag to disable default metric. Set
+ to >0 to disable. [default: 0]
+ -eta,--learning_rate <arg> Step size shrinkage used in update
+ to prevents overfitting [default:
+ 0.3]
+ -eval_metric <arg> Evaluation metrics for validation
+ data. A default metric is assigned
+ according to the objective:
+ - rmse: for regression
+ - error: for classification
+ - map: for ranking
+ For a list of valid inputs, see
+ XGBoost Parameters.
+ -feature_selector <arg> Feature selection and ordering
+ method. [Choices: cyclic (default),
+ shuffle, random, greedy, thrifty]
+ -gamma,--min_split_loss <arg> Minimum loss reduction required to
+ make a further partition on a leaf
+ node of the tree. [default: 0.0]
+ -grow_policy <arg> Controls a way new nodes are added
+ to the tree. Currently supported
+ only if tree_method is set to hist.
+ [default: depthwise, Choices:
+ depthwise, lossguide]
+ -lambda,--reg_lambda <arg> L2 regularization term on weights.
+ Increasing this value will make
+ model more conservative. [default:
+ 1.0 for gbtree, 0.0 for gblinear]
+ -lambda_bias <arg> L2 regularization term on bias
+ [default: 0.0]
+ -max_bin <arg> Maximum number of discrete bins to
+ bucket continuous features. Only
+ used if tree_method is set to hist.
+ [default: 256]
+ -max_delta_step <arg> Maximum delta step we allow each
+ tree's weight estimation to be
+ [default: 0]
+ -max_depth <arg> Max depth of decision tree [default:
+ 6]
+ -max_leaves <arg> Maximum number of nodes to be added.
+ Only relevant when
+ grow_policy=lossguide is set.
+ [default: 0]
+ -maximize_evaluation_metrics <arg> Maximize evaluation metrics
+ [default: false]
+ -min_child_weight <arg> Minimum sum of instance weight
+ (hessian) needed in a child
+ [default: 1.0]
+ -normalize_type <arg> Type of normalization algorithm.
+ [Choices: tree (default), forest]
+ -num_class <arg> Number of classes to classify
+ -num_early_stopping_rounds <arg> Minimum rounds required for early
+ stopping [default: 0]
+ -num_feature <arg> Feature dimension used in boosting
+ [default: set automatically by
+ xgboost]
+ -num_parallel_tree <arg> Number of parallel trees constructed
+ during each iteration. This option
+ is used to support boosted random
+ forest. [default: 1]
+ -num_pbuffer <arg> Size of prediction buffer [default:
+ set automatically by xgboost]
+ -num_round,--iters <arg> Number of boosting iterations
+ [default: 10]
+ -objective <arg> Specifies the learning task and the
+ corresponding learning objective.
+ Examples: reg:linear, reg:logistic,
+ multi:softmax. For a full list of
+ valid inputs, refer to XGBoost
+ Parameters. [default: reg:linear]
+ -one_drop <arg> When this flag is enabled, at least
+ one tree is always dropped during
+ the dropout. 0 or 1. [default: 0]
+ -process_type <arg> A type of boosting process to run.
+ [Choices: default, update]
+ -rate_drop <arg> Dropout rate in range [0.0, 1.0].
+ [default: 0.0]
+ -refresh_leaf <arg> This is a parameter of the refresh
+ updater plugin. When this flag is 1,
+ tree leafs as well as tree nodes’
+ stats are updated. When it is 0,
+ only node stats are updated.
+ [default: 1]
+ -sample_type <arg> Type of sampling algorithm.
+ [Choices: uniform (default),
+ weighted]
+ -scale_pos_weight <arg> ontrol the balance of positive and
+ negative weights, useful for
+ unbalanced classes. A typical value
+ to consider: sum(negative instances)
+ / sum(positive instances) [default:
+ 1.0]
+ -seed <arg> Random number seed. [default: 43]
+ -silent <arg> Deprecated. Please use verbosity
+ instead. 0 means printing running
+ messages, 1 means silent mode
+ [default: 1]
+ -sketch_eps <arg> This roughly translates into O(1 /
+ sketch_eps) number of bins.
+ Compared to directly select number
+ of bins, this comes with theoretical
+ guarantee with sketch accuracy.
+ Only used for tree_method=approx.
+ Usually user does not have to tune
+ this. [default: 0.03]
+ -skip_drop <arg> Probability of skipping the dropout
+ procedure during a boosting
+ iteration in range [0.0, 1.0].
+ [default: 0.0]
+ -subsample <arg> Subsample ratio of the training
+ instance in range (0.0,1.0]
+ [default: 1.0]
+ -top_k <arg> The number of top features to select
+ in greedy and thrifty feature
+ selector. The value of 0 means using
+ all the features. [default: 0]
+ -tree_method <arg> The tree construction algorithm used
+ in XGBoost. [default: auto, Choices:
+ auto, exact, approx, hist]
+ -tweedie_variance_power <arg> Parameter that controls the variance
+ of the Tweedie distribution in range
+ [1.0, 2.0]. [default: 1.5]
+ -updater <arg> A comma-separated string that
+ defines the sequence of tree
+ updaters to run. For a full list of
+ valid inputs, please refer to
+ XGBoost Parameters. [default:
+ 'grow_colmaker,prune' for gbtree,
+ 'shotgun' for gblinear]
+ -validation_ratio <arg> Validation ratio in range [0.0,1.0]
+ [default: 0.2]
+ -verbosity <arg> Verbosity of printing messages.
+ Choices: 0 (silent), 1 (warning), 2
+ (info), 3 (debug). [default: 0]
+```
+
+Objective function `-objective` SHOULD be specified though `-objective reg:linear` is used for Objective function by the default.
+For the full list of objective functions, please refer [this xgboost v0.90 documentation](https://xgboost.readthedocs.io/en/stable/parameter.html#learning-task-parameters).
+
+The following objectives would widely be used for regression, binary classication, and multiclass classication, respectively.
+
+- `reg:squarederror` regression with squared loss.
+- `binary:logistic` logistic regression for binary classification, output probability.
+- `binary:hinge` hinge loss for binary classification. This makes predictions of 0 or 1, rather than producing probabilities.
+- `multi:softmax` set XGBoost to do multiclass classification using the softmax objective, you also need to set `num_class` (number of classes).
+- `multi:softprob` same as softmax, but output a vector of `ndata * nclass`, which can be further reshaped to `ndata * nclass` matrix. The result contains predicted probability of each data point belonging to each class.
+
+Other hyperparameters better to be tuned are:
+
+- `-booster gbree` Which booster to use. The default gbtree (Gradient Boosting Trees) would be fine for most cases. Can be `gbtree`, `gblinear` or `dart`; gbtree and dart use tree based models while gblinear uses linear functions.
+- `-eta 0.1` The learning rate, 0.3 by the default. 0.05, 0.1, 0.3 are worth trying.
+- `-max_depth 6` The maximum depth of the tree. The default value 6 would be fine for most case. Recommended value range is 5-10.
+- `-num_class 3` The number of classes MUST be specified for multiclass classification (i.e., `-objective multi:softmax` or `-objective multi:softprob`)
+- `-num_round 10` The number of rounds for boosting. 10 or more would be preferred.
+- `-num_early_stopping_rounds 3` The number of rounds required for early stopping. Without specifying `-num_early_stopping_rounds`, no early stopping is NOT carried. When `-num_round=100` and `-num_early_stopping_rounds=5`, traning could be early stopped at 15th iteration if there is no evaluation result greater than the 10th iteration's (best one). Early stopping 3 or so would be preferred.
+- `-validation_ratio 0.2` The ratio data used for validation (early stopping). 0.2 would be enough for most cases. Note that 80% data is used for training when `validation_ratio 0.2` is set.
+
+You can find the underlying XGBoost version by:
+
+```sql
+select xgboost_version();
+> 0.90
+```
+
+## Training
+
+`train_xgboost` UDTF is used for training.
+
+The function signature is `train_xgboost(array<string|double> features, double target [,string options])` and it returns a prediction model as a relation consist of `<string model_id, array<string> pred_model>`.
+
+```sql
+-- explicitly use 3 reducers
+-- set mapred.reduce.tasks=3;
+
+drop table xgb_lr_model;
+create table xgb_lr_model as
+select
+ train_xgboost(features, label, '-objective binary:logistic -num_round 10 -num_early_stopping_rounds 3')
+ as (model_id, model)
+from (
+ select features, label
+ from news20b_train
+ cluster by rand(43) -- shuffle data to reducers
+) shuffled;
+
+drop table xgb_hinge_model;
+create table xgb_hinge_model as
+select
+ train_xgboost(features, label, '-objective binary:hinge -num_round 10 -num_early_stopping_rounds 3')
+ as (model_id, model)
+from (
+ select features, label
+ from news20b_train
+ cluster by rand(43) -- shuffle data to reducers
+) shuffled;
+```
+
+> #### Caution
+> `cluster by rand()` is NOT required when training data is small and a single task is launched for XGBoost training.
+> `cluster by rand()` shuffles data at random and divided it for multiple XGBoost instances.
+
+## prediction
+
+```sql
+drop table xgb_lr_predicted;
+create table xgb_lr_predicted
+as
+select
+ rowid,
+ array_avg(predicted) as predicted,
+ avg(predicted[0]) as prob
+from (
+ select
+ -- fast predictition by xgboost-predictor-java (https://github.com/komiya-atsushi/xgboost-predictor-java/)
+ xgboost_predict(rowid, features, model_id, model) as (rowid, predicted)
+ -- predict by xgboost4j (https://xgboost.readthedocs.io/en/stable/jvm/)
+ -- xgboost_batch_predict(rowid, features, model_id, model) as (rowid, predicted)
+ from
+ -- for each model l
+ -- for each test r
+ -- predict
+ xgb_lr_model l
+ LEFT OUTER JOIN news20b_test r
+) t
+group by rowid;
+
+drop table xgb_hinge_predicted;
+create table xgb_hinge_predicted
+as
+select
+ rowid,
+ -- voting
+ -- if(sum(if(predicted[0]=1,1,0)) > sum(if(predicted[0]=0,1,0)),1,-1) as predicted
+ majority_vote(if(predicted[0]=1, 1, -1)) as predicted
+from (
+ select
+ -- binary:hinge is not supported in xgboost_predict
+ -- binary:hinge returns [1.0] or [0.0] for predicted
+ xgboost_batch_predict(rowid, features, model_id, model)
+ as (rowid, predicted)
+ from
+ -- for each model l
+ -- for each test r
+ -- predict
+ xgb_hinge_model l
+ LEFT OUTER JOIN news20b_test r
+) t
+group by
+ rowid
+```
+
+You can find the function signature of `xgboost_predict` by
+
+```sql
+select xgboost_predict();
+
+usage: xgboost_predict(PRIMITIVE rowid, array<string|double> features,
+ string model_id, array<string> pred_model [, string options]) -
+ Returns a prediction result as (string rowid, array<double>
+ predicted)
+
+select xgboost_batch_predict();
+
+usage: xgboost_batch_predict(PRIMITIVE rowid, array<string|double>
+ features, string model_id, array<string> pred_model [, string
+ options]) - Returns a prediction result as (string rowid,
+ array<double> predicted) [-batch_size <arg>]
+ -batch_size <arg> Number of rows to predict together [default: 128]
+```
+
+> #### Caution
+> `xgboost_predict` outputs probability for `-objective binary:logistic` while 0/1 is resulted for `-objective binary:hinge`.
+>
+> `xgboost_predict` only support the following models and objectives because it uses [xgboost-predictor-java](https://github.com/komiya-atsushi/xgboost-predictor-java):
+> Models: {gblinear, gbtree, dart}
+> Objective functions: {binary:logistic, binary:logitraw, multi:softmax, multi:softprob, reg:linear, reg:squarederror, rank:pairwise}
+>
+> For other models and objectives, please use `xgboost_batch_predict` that uses [xgboost4j](https://xgboost.readthedocs.io/en/stable/jvm/) insead.
+
+## evaluation
+
+```sql
+WITH submit as (
+ select
+ t.label as actual,
+ -- probability thresholding by 0.5
+ if(p.prob > 0.5,1,-1) as predicted
+ from
+ news20b_test t
+ JOIN xgb_lr_predicted p
+ on (t.rowid = p.rowid)
+)
+select
+ sum(if(actual = predicted, 1, 0)) / count(1) as accuracy
+from
+ submit;
+```
+
+> 0.8372698158526821 (logistic loss)
+
+```sql
+WITH submit as (
+ select
+ t.label as actual,
+ p.predicted
+ from
+ news20b_test t
+ JOIN xgb_hinge_predicted p
+ on (t.rowid = p.rowid)
+)
+select
+ sum(if(actual=predicted,1,0)) / count(1) as accuracy
+from
+ submit;
+```
+
+> 0.7752201761409128 (hinge loss)
+
diff --git a/docs/gitbook/misc/funcs.md b/docs/gitbook/misc/funcs.md
index 0f19ff1..bda8e22 100644
--- a/docs/gitbook/misc/funcs.md
+++ b/docs/gitbook/misc/funcs.md
@@ -636,15 +636,80 @@ Reference: <a href="https://papers.nips.cc/paper/3848-adaptive-regularization-of
# XGBoost
-- `train_multiclass_xgboost_classifier(string[] features, double target [, string options])` - Returns a relation consisting of <string model_id, array<byte> pred_model>
+- `train_xgboost(array<string|double> features, int|double target [, string options])` - Returns a relation consists of <string model_id, array<string> pred_model>
+ ```sql
+ SELECT
+ train_xgboost(features, label, '-objective binary:logistic -iters 10')
+ as (model_id, model)
+ from (
+ select features, label
+ from xgb_input
+ cluster by rand(43) -- shuffle
+ ) shuffled;
+ ```
-- `train_xgboost_classifier(array<string> features, double target [, string options])` - Returns a relation consisting of <string model_id, array<byte> pred_model>
+- `xgboost_batch_predict(PRIMITIVE rowid, array<string|double> features, string model_id, array<string> pred_model [, string options])` - Returns a prediction result as (string rowid, array<double> predicted)
+ ```sql
+ select
+ rowid,
+ array_avg(predicted) as predicted,
+ avg(predicted[0]) as predicted0
+ from (
+ select
+ xgboost_batch_predict(rowid, features, model_id, model) as (rowid, predicted)
+ from
+ xgb_model l
+ LEFT OUTER JOIN xgb_input r
+ ) t
+ group by rowid;
+ ```
-- `train_xgboost_regr(string[] features, double target [, string options])` - Returns a relation consisting of <string model_id, array<byte> pred_model>
+- `xgboost_predict(PRIMITIVE rowid, array<string|double> features, string model_id, array<string> pred_model [, string options])` - Returns a prediction result as (string rowid, array<double> predicted)
+ ```sql
+ select
+ rowid,
+ array_avg(predicted) as predicted,
+ avg(predicted[0]) as predicted0
+ from (
+ select
+ xgboost_predict(rowid, features, model_id, model) as (rowid, predicted)
+ from
+ xgb_model l
+ LEFT OUTER JOIN xgb_input r
+ ) t
+ group by rowid;
+ ```
-- `xgboost_multiclass_predict(string rowid, string[] features, string model_id, array<byte> pred_model [, string options])` - Returns a prediction result as (string rowid, string label, float probability)
+- `xgboost_predict_one(PRIMITIVE rowid, array<string|double> features, string model_id, array<string> pred_model [, string options])` - Returns a prediction result as (string rowid, double predicted)
+ ```sql
+ select
+ rowid,
+ avg(predicted) as predicted
+ from (
+ select
+ xgboost_predict_one(rowid, features, model_id, model) as (rowid, predicted)
+ from
+ xgb_model l
+ LEFT OUTER JOIN xgb_input r
+ ) t
+ group by rowid;
+ ```
-- `xgboost_predict(string rowid, string[] features, string model_id, array<byte> pred_model [, string options])` - Returns a prediction result as (string rowid, float predicted)
+- `xgboost_predict_triple(PRIMITIVE rowid, array<string|double> features, string model_id, array<string> pred_model [, string options])` - Returns a prediction result as (string rowid, string label, double probability)
+ ```sql
+ select
+ rowid,
+ label,
+ avg(prob) as prob
+ from (
+ select
+ xgboost_predict_triple(rowid, features, model_id, model) as (rowid, label, prob)
+ from
+ xgb_model l
+ LEFT OUTER JOIN xgb_input r
+ ) t
+ group by rowid, label;
+ ```
- `xgboost_version()` - Returns the version of xgboost
```sql
diff --git a/docs/gitbook/multiclass/iris_xgboost.md b/docs/gitbook/multiclass/iris_xgboost.md
new file mode 100644
index 0000000..3eb3560
--- /dev/null
+++ b/docs/gitbook/multiclass/iris_xgboost.md
@@ -0,0 +1,146 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<!-- toc -->
+
+## Data preparation
+
+```sql
+create database iris;
+use iris;
+
+create external table raw (
+ sepal_length int,
+ sepal_width int,
+ petal_length int,
+ petak_width int,
+ class string
+)
+ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY ','
+ LINES TERMINATED BY '\n'
+STORED AS TEXTFILE LOCATION '/dataset/iris/raw';
+
+$ sed '/^$/d' iris.data | hadoop fs -put - /dataset/iris/raw/iris.data
+```
+
+```sql
+drop table label_mapping;
+create table label_mapping
+as
+select
+ class,
+ rank - 1 as label -- zero start index
+from (
+ select
+ distinct class,
+ dense_rank() over (order by class) as rank
+ from
+ raw
+) t;
+```
+
+```sql
+drop table xgb_input;
+create table xgb_input
+as
+select
+ rowid() as rowid,
+ array(sepal_length, sepal_width, petal_length, petal_width) as dense_features,
+ indexed_features(sepal_length, sepal_width, petal_length, petal_width) as sparse_features,
+ t2.label
+from
+ raw t1
+ JOIN label_mapping t2 ON (t1.class = t2.class)
+;
+
+select * from xgb_input limit 3;
+```
+
+| xgb_input.rowid | xgb_input.dense_features | xgb_input.sparse_features | xgb_input.label |
+|:-:|:-:|:-:|:-:|
+| 1-1 | [5,3,1,0] | ["1:5","2:3","3:1","4:0"] | 0 |
+| 1-2 | [4,3,1,0] | ["1:4","2:3","3:1","4:0"] | 0 |
+| 1-3 | [4,3,1,0] | ["1:4","2:3","3:1","4:0"] | 0 |
+
+## Training
+
+```
+-- explicitly use 3 reducers
+-- set mapred.reduce.tasks=3;
+
+drop table xgb_softmax_model;
+create table xgb_softmax_model
+as
+select
+ train_xgboost(features, label, '-objective multi:softmax -num_class 3 -num_round 10 -num_early_stopping_rounds 3')
+ as (model_id, model)
+from (
+ select
+ -- both sparse and dense format is supported
+ dense_features as features, label
+ -- sparse_features as features, label
+ from
+ xgb_input
+ cluster by rand(43) -- shuffle
+) shuffled;
+```
+
+> #### Caution
+> `-num_class` is required for multiclass objectives.
+> Note both sparse and dense vector is supported for feature vector format.
+
+## Prediction
+
+```sql
+drop table xgb_softmax_predicted;
+create table xgb_softmax_predicted as
+select
+ rowid,
+ majority_vote(cast(predicted as int)) as label
+from (
+ select
+ xgboost_predict_one(rowid, dense_features, model_id, model) as (rowid, predicted)
+ -- xgboost_predict_one(rowid, sparse_features, model_id, model) as (rowid, predicted)
+ from
+ xgb_softmax_model l
+ LEFT OUTER JOIN xgb_input r
+) t
+group by rowid;
+```
+
+## Evaluation
+
+```sql
+WITH validate as (
+ select
+ t.label as actual,
+ p.label as predicted
+ from
+ xgb_input t
+ JOIN xgb_softmax_predicted p
+ on (t.rowid = p.rowid)
+)
+select
+ sum(if(actual=predicted,1.0,0.0))/count(1)
+from
+ validate;
+```
+
+> 0.9533333333333333333
\ No newline at end of file
diff --git a/docs/gitbook/multiclass/news20_dataset.md b/docs/gitbook/multiclass/news20_dataset.md
index 0ba0360..ddbaa27 100644
--- a/docs/gitbook/multiclass/news20_dataset.md
+++ b/docs/gitbook/multiclass/news20_dataset.md
@@ -16,7 +16,7 @@
specific language governing permissions and limitations
under the License.
-->
-
+
Get the news20 dataset.
https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass.html#news20
@@ -52,11 +52,6 @@ hadoop fs -copyFromLocal news20_scale.test /dataset/news20-multiclass/test
```sql
use news20;
-delete jar /home/myui/tmp/hivemall.jar;
-add jar /home/myui/tmp/hivemall.jar;
-
-source /home/myui/tmp/define-all.hive;
-
Create external table news20mc_train (
rowid int,
label int,
diff --git a/docs/gitbook/multiclass/news20_pa.md b/docs/gitbook/multiclass/news20_pa.md
index c57d08d..1543ecc 100644
--- a/docs/gitbook/multiclass/news20_pa.md
+++ b/docs/gitbook/multiclass/news20_pa.md
@@ -16,25 +16,13 @@
specific language governing permissions and limitations
under the License.
-->
-
-Preparation
-=========
-## UDF preparation
-```
-delete jar /home/myui/tmp/hivemall.jar;
-add jar /home/myui/tmp/hivemall.jar;
-
-source /home/myui/tmp/define-all.hive;
-```
+<!-- toc -->
----
-#[Passive Aggressive (PA2)]
+# Passive Aggressive (PA2)
-Training
-======
+## Training
-## model building
```sql
drop table news20mc_pa2_model1;
create table news20mc_pa2_model1 as
@@ -51,7 +39,7 @@ from
group by label, feature;
```
-## prediction
+## Prediction
```
create or replace view news20mc_pa2_predict1
as
@@ -78,7 +66,7 @@ group by rowid
) t2;
```
-## evaluation
+## Evaluation
```sql
create or replace view news20mc_pa2_submit1 as
select
@@ -99,11 +87,3 @@ where actual == predicted;
> 0.8204357625845229 (x3)
> 0.8204357625845229 (x3 + bagging)
-
-## Cleaning
-
-```sql
-drop table news20mc_pa2_model1;
-drop table news20mc_pa2_predict1;
-drop view news20mc_pa2_submit1;
-```
diff --git a/docs/gitbook/multiclass/news20_scw.md b/docs/gitbook/multiclass/news20_scw.md
index fbe5153..4f3e49b 100644
--- a/docs/gitbook/multiclass/news20_scw.md
+++ b/docs/gitbook/multiclass/news20_scw.md
@@ -16,29 +16,10 @@
specific language governing permissions and limitations
under the License.
-->
-
-| Algorithm | Accuracy |
-|:-----------|------------:|
-| PA2 | 0.8204357625845229 |
-| SCW1 | 0.8314550463310794 |
-| AROW | 0.8474830954169797 |
-| SCW2 | 0.8482344102178813 |
-| CW | 0.850488354620586 |
----
-
-Preparation
-=========
-## UDF preparation
-```sql
-delete jar /home/myui/tmp/hivemall.jar;
-add jar /home/myui/tmp/hivemall.jar;
-
-source /home/myui/tmp/define-all.hive;
-```
+<!-- toc -->
----
-#[CW]
+# CW
## training
```sql
@@ -104,16 +85,7 @@ where actual == predicted;
> 0.850488354620586
-## Cleaning
-
-```sql
-drop table news20mc_cw_model1;
-drop table news20mc_cw_predict1;
-drop view news20mc_cw_submit1;
-```
-
----
-#[AROW]
+# AROW
## training
```sql
@@ -179,18 +151,10 @@ where actual == predicted;
> 0.8474830954169797
-## Cleaning
-
-```sql
-drop table news20mc_arow_model1;
-drop table news20mc_arow_predict1;
-drop view news20mc_arow_submit1;
-```
-
----
-#[SCW1]
+# SCW1
## training
+
```sql
drop table news20mc_scw_model1;
create table news20mc_scw_model1 as
@@ -210,6 +174,7 @@ group by label, feature;
```
## prediction
+
```sql
create or replace view news20mc_scw_predict1
as
@@ -237,6 +202,7 @@ group by rowid
```
## evaluation
+
```sql
create or replace view news20mc_scw_submit1 as
select
@@ -254,18 +220,10 @@ where actual == predicted;
> 0.8314550463310794
-## Cleaning
-
-```sql
-drop table news20mc_scw_model1;
-drop table news20mc_scw_predict1;
-drop view news20mc_scw_submit1;
-```
-
----
-#[SCW2]
+# SCW2
## training
+
```sql
drop table news20mc_scw2_model1;
create table news20mc_scw2_model1 as
@@ -285,6 +243,7 @@ group by label, feature;
```
## prediction
+
```sql
create or replace view news20mc_scw2_predict1
as
@@ -312,6 +271,7 @@ group by rowid
```
## evaluation
+
```sql
create or replace view news20mc_scw2_submit1 as
select
@@ -329,10 +289,12 @@ where actual == predicted;
> 0.8482344102178813
-## Cleaning
+# Wrap up of evaluation results
-```sql
-drop table news20mc_scw2_model1;
-drop table news20mc_scw2_predict1;
-drop view news20mc_scw2_submit1;
-```
+| Algorithm | Accuracy |
+|:-----------|------------:|
+| PA2 | 0.8204357625845229 |
+| SCW1 | 0.8314550463310794 |
+| AROW | 0.8474830954169797 |
+| SCW2 | 0.8482344102178813 |
+| CW | 0.850488354620586 |
diff --git a/docs/gitbook/multiclass/news20_xgboost.md b/docs/gitbook/multiclass/news20_xgboost.md
new file mode 100644
index 0000000..46d9dde
--- /dev/null
+++ b/docs/gitbook/multiclass/news20_xgboost.md
@@ -0,0 +1,137 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<!-- toc -->
+
+## training
+
+For multiclass classification, the following two objects are supported in xgboost:
+
+- `multi:softmax` set XGBoost to do multiclass classification using the softmax objective, you also need to set `num_class` (number of classes).
+- `multi:softprob` same as softmax, but output a vector of `ndata * nclass`, which can be further reshaped to `ndata * nclass` matrix. The result contains predicted probability of each data point belonging to each class.
+
+```sql
+select count(distinct label) from news20mc_train;
+> 20
+
+-- explicitly use 3 reducers
+-- set mapred.reduce.tasks=3;
+
+drop table xgb_softmax_model;
+create table xgb_softmax_model as
+select
+ train_xgboost(features, label, '-objective multi:softmax -num_class 20 -num_round 10 -num_early_stopping_rounds 3')
+ as (model_id, model)
+from (
+ select features, (label - 1) as label
+ from news20mc_train
+ cluster by rand(43) -- shuffle data to reducers
+) shuffled;
+
+drop table xgb_softprob_model;
+create table xgb_softprob_model as
+select
+ train_xgboost(features, label, '-objective multi:softprob -num_class 20 -num_round 10 -num_early_stopping_rounds 3')
+ as (model_id, model)
+from (
+ select features, (label - 1) as label
+ from news20mc_train
+ cluster by rand(43) -- shuffle data to reducers
+) shuffled;
+```
+
+> #### Caution
+> `-num_class` is required for multiclass objectives.
+> The target label must be in range `[0, num_class)` (i.e., 0, 1, 2, .., num_class-1).
+
+## prediction
+
+```sql
+drop table xgb_softmax_predicted;
+create table xgb_softmax_predicted as
+select
+ rowid,
+ majority_vote(cast(predicted as int) + 1) as label
+from (
+ select
+ xgboost_predict_one(rowid, features, model_id, model) as (rowid, predicted)
+ from
+ xgb_softmax_model l
+ LEFT OUTER JOIN news20mc_test r
+) t
+group by rowid;
+
+
+drop table xgb_softprob_predicted;
+create table xgb_softprob_predicted as
+select
+ rowid,
+ array_avg(predicted) as prob,
+ argmax(array_avg(predicted)) + 1 as label -- convert 0 start index to 1 start index
+from (
+ select
+ xgboost_predict(rowid, features, model_id, model) as (rowid, predicted)
+ from
+ xgb_softprob_model l
+ LEFT OUTER JOIN news20mc_test r
+) t
+group by
+ rowid;
+```
+
+> #### Caution
+> For `-objective softmax`, xgboost predictor returns class label in double.
+> For `-objective softprob`, probabilities for each label is returned in `array<double>`.
+
+## evaluation
+
+```sql
+WITH validate as (
+ select
+ t.label as actual,
+ p.label as predicted
+ from
+ news20mc_test t
+ JOIN xgb_softmax_predicted p
+ on (t.rowid = p.rowid)
+)
+select
+ sum(if(actual=predicted,1.0,0.0))/count(1)
+from
+ validate;
+
+WITH validate as (
+ select
+ t.label as actual,
+ p.label as predicted
+ from
+ news20mc_test t
+ JOIN xgb_softprob_predicted p
+ on (t.rowid = p.rowid)
+)
+select
+ sum(if(actual=predicted,1.0,0.0))/count(1)
+from
+ validate;
+```
+
+|objective| accuracy|
+|:-:|:-:|
+| softmax | 0.6689206110693713999 |
+| softprob | 0.6944653143000250438 |
\ No newline at end of file
diff --git a/docs/gitbook/regression/e2006_xgboost.md b/docs/gitbook/regression/e2006_xgboost.md
new file mode 100644
index 0000000..2d4e1bf
--- /dev/null
+++ b/docs/gitbook/regression/e2006_xgboost.md
@@ -0,0 +1,107 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+This tutorial explains how to use XGBoost for regression problems.
+
+<!-- toc -->
+
+## Training
+
+The following objective is supported in XGboost for regression:
+
+- `reg:squarederror` regression with squared loss
+- `reg:logistic` logistic regression
+
+`reg:squarederror` is widely used as the regression objective.
+
+```sql
+use e2006;
+
+desc e2006tfidf_train;
+```
+
+| col_name | data_type |
+|:-:|:-:|
+| rowid | int |
+| target | float |
+| features | array<string> |
+
+
+```sql
+-- explicitly use 3 reducers
+-- set mapred.reduce.tasks=3
+
+drop table xgb_regr_model;
+create table xgb_regr_model as
+select
+ train_xgboost(features, target, '-objective reg:squarederror -num_round 10 -num_early_stopping_rounds 3')
+ as (model_id, model)
+from (
+ select features, target
+ from e2006tfidf_train
+ cluster by rand(43) -- shuffle data to reducers
+) shuffled;
+```
+
+## prediction
+
+```sql
+drop table xgb_regr_predicted;
+create table xgb_regr_predicted as
+select
+ rowid,
+ avg(predicted) as predicted
+from (
+ select
+ xgboost_predict_one(rowid, features, model_id, model) as (rowid, predicted)
+ from
+ xgb_regr_model l
+ LEFT OUTER JOIN e2006tfidf_test r
+) t
+group by rowid;
+```
+
+> #### Note
+> `xgboost_predict` returns new double[1] (e.g., [-3.9760303385555744]) for `-objective reg:squarederror`.
+> On the other hand, `xgboost_predict_one` returns a scalar double value as predicted `-3.9760303385555744`.
+
+## evaluation
+
+```sql
+WITH submit as (
+ select
+ t.target as actual,
+ p.predicted as predicted
+ from
+ e2006tfidf_test t
+ JOIN xgb_regr_predicted p
+ on (t.rowid = p.rowid)
+)
+select
+ rmse(predicted, actual) as RMSE,
+ mse(predicted, actual) as MSE,
+ mae(predicted, actual) as MAE,
+ r2(predicted, actual) as R2
+from
+ submit;
+```
+
+| rmse | mse | mae | r2 |
+|:-:|:-:|:-:|:-:|
+| 0.3949633797136429 | 0.15599607131482326 | 0.25367043577533693 | 0.4603881976325721 |
diff --git a/pom.xml b/pom.xml
index ea15205..ec30d1c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -303,6 +303,10 @@
<enabled>true</enabled>
</releases>
</repository>
+ <repository>
+ <id>jcenter</id>
+ <url>https://jcenter.bintray.com/</url>
+ </repository>
</repositories>
<issueManagement>
diff --git a/resources/ddl/define-additional.hive b/resources/ddl/define-additional.hive
index d08ac59..802245d 100644
--- a/resources/ddl/define-additional.hive
+++ b/resources/ddl/define-additional.hive
@@ -19,20 +19,21 @@ create temporary function stoptags_exclude as 'hivemall.nlp.tokenizer.StoptagsEx
-- XGBoost related features --
------------------------------
-drop temporary function if exists train_xgboost_regr;
-create temporary function train_xgboost_regr as 'hivemall.xgboost.regression.XGBoostRegressionUDTF';
-
-drop temporary function if exists train_xgboost_classifier;
-create temporary function train_xgboost_classifier as 'hivemall.xgboost.classification.XGBoostBinaryClassifierUDTF';
+drop temporary function if exists xgboost_version;
+create temporary function xgboost_version as 'hivemall.xgboost.XGBoostVersionUDF';
-drop temporary function if exists train_multiclass_xgboost_classifier;
-create temporary function train_multiclass_xgboost_classifier as 'hivemall.xgboost.classification.XGBoostMulticlassClassifierUDTF';
+drop temporary function if exists train_xgboost;
+create temporary function train_xgboost as 'hivemall.xgboost.XGBoostTrainUDTF';
drop temporary function if exists xgboost_predict;
-create temporary function xgboost_predict as 'hivemall.xgboost.tools.XGBoostPredictUDTF';
+create temporary function xgboost_predict as 'hivemall.xgboost.XGBoostOnlinePredictUDTF';
-drop temporary function if exists xgboost_multiclass_predict;
-create temporary function xgboost_multiclass_predict as 'hivemall.xgboost.tools.XGBoostMulticlassPredictUDTF';
+drop temporary function if exists xgboost_batch_predict;
+create temporary function xgboost_batch_predict as 'hivemall.xgboost.XGBoostBatchPredictUDTF';
+
+drop temporary function if exists xgboost_predict_one;
+create temporary function xgboost_predict_one as 'hivemall.xgboost.XGBoostPredictOneUDTF';
+
+drop temporary function if exists xgboost_predict_triple;
+create temporary function xgboost_predict_triple as 'hivemall.xgboost.XGBoostPredictTripleUDTF';
-drop temporary function if exists xgboost_version;
-create temporary function xgboost_version as 'hivemall.xgboost.XGBoostVersionUDF';
diff --git a/resources/ddl/define-all-as-permanent.hive b/resources/ddl/define-all-as-permanent.hive
index c8bf307..82b12dc 100644
--- a/resources/ddl/define-all-as-permanent.hive
+++ b/resources/ddl/define-all-as-permanent.hive
@@ -900,20 +900,20 @@ CREATE FUNCTION majority_vote as 'hivemall.tools.aggr.MajorityVoteUDAF' USING JA
-- XGBoost related features --
------------------------------
-DROP FUNCTION train_xgboost_regr;
-CREATE FUNCTION train_xgboost_regr AS 'hivemall.xgboost.regression.XGBoostRegressionUDTF' USING JAR '${hivemall_jar}';
-
-DROP FUNCTION train_xgboost_classifier;
-CREATE FUNCTION train_xgboost_classifier AS 'hivemall.xgboost.classification.XGBoostBinaryClassifierUDTF' USING JAR '${hivemall_jar}';
+DROP FUNCTION IF EXISTS xgboost_version;
+CREATE FUNCTION xgboost_version as 'hivemall.xgboost.XGBoostVersionUDF' USING JAR '${hivemall_jar}';
-DROP FUNCTION train_multiclass_xgboost_classifier;
-CREATE FUNCTION train_multiclass_xgboost_classifier AS 'hivemall.xgboost.classification.XGBoostMulticlassClassifierUDTF' USING JAR '${hivemall_jar}';
+DROP FUNCTION train_xgboost;
+CREATE FUNCTION train_xgboost AS 'hivemall.xgboost.XGBoostTrainUDTF' USING JAR '${hivemall_jar}';
DROP FUNCTION xgboost_predict;
-CREATE FUNCTION xgboost_predict AS 'hivemall.xgboost.tools.XGBoostPredictUDTF' USING JAR '${hivemall_jar}';
+CREATE FUNCTION xgboost_predict AS 'hivemall.xgboost.XGBoostOnlinePredictUDTF' USING JAR '${hivemall_jar}';
-DROP FUNCTION xgboost_multiclass_predict;
-CREATE FUNCTION xgboost_multiclass_predict AS 'hivemall.xgboost.tools.XGBoostMulticlassPredictUDTF' USING JAR '${hivemall_jar}';
+DROP FUNCTION xgboost_batch_predict;
+CREATE FUNCTION xgboost_batch_predict AS 'hivemall.xgboost.XGBoostBatchPredictUDTF' USING JAR '${hivemall_jar}';
-DROP FUNCTION IF EXISTS xgboost_version;
-CREATE FUNCTION xgboost_version as 'hivemall.xgboost.XGBoostVersionUDF' USING JAR '${hivemall_jar}';
+DROP FUNCTION xgboost_predict_one;
+CREATE FUNCTION xgboost_predict_one AS 'hivemall.xgboost.XGBoostPredictOneUDTF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION xgboost_predict_triple;
+CREATE FUNCTION xgboost_predict_triple AS 'hivemall.xgboost.XGBoostPredictTripleUDTF' USING JAR '${hivemall_jar}';
diff --git a/tools/hivemall-docs/src/main/java/hivemall/docs/FuncsListGeneratorMojo.java b/tools/hivemall-docs/src/main/java/hivemall/docs/FuncsListGeneratorMojo.java
index 3e7bbdb..bf9babc 100644
--- a/tools/hivemall-docs/src/main/java/hivemall/docs/FuncsListGeneratorMojo.java
+++ b/tools/hivemall-docs/src/main/java/hivemall/docs/FuncsListGeneratorMojo.java
@@ -143,8 +143,7 @@ public class FuncsListGeneratorMojo extends AbstractMojo {
funcsHeaders.put("## Bagging", Collections.singletonList("hivemall.ensemble.bagging"));
funcsHeaders.put("# Decision trees and RandomForest", Arrays.asList(
"hivemall.smile.classification", "hivemall.smile.regression", "hivemall.smile.tools"));
- funcsHeaders.put("# XGBoost", Arrays.asList("hivemall.xgboost.classification",
- "hivemall.xgboost.regression", "hivemall.xgboost.tools", "hivemall.xgboost"));
+ funcsHeaders.put("# XGBoost", Arrays.asList("hivemall.xgboost"));
funcsHeaders.put("# Term Vector Model", Collections.singletonList("hivemall.ftvec.text"));
funcsHeaders.put("# Others",
Arrays.asList("hivemall", "hivemall.dataset", "hivemall.ftvec.text"));
diff --git a/xgboost/pom.xml b/xgboost/pom.xml
index f124102..8cce245 100644
--- a/xgboost/pom.xml
+++ b/xgboost/pom.xml
@@ -16,7 +16,9 @@
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -32,7 +34,7 @@
<properties>
<main.basedir>${project.parent.basedir}</main.basedir>
- <xgboost.version>0.90-rc1</xgboost.version>
+ <xgboost.version>0.90-rc2</xgboost.version>
</properties>
<dependencies>
@@ -100,6 +102,12 @@
<version>2.21</version>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>biz.k11i</groupId>
+ <artifactId>xgboost-predictor</artifactId>
+ <version>0.3.1</version>
+ <scope>compile</scope>
+ </dependency>
<!-- test scope -->
<dependency>
diff --git a/xgboost/src/main/java/hivemall/xgboost/XGBoostBaseUDTF.java b/xgboost/src/main/java/hivemall/xgboost/XGBoostBaseUDTF.java
deleted file mode 100644
index 8d77ae3..0000000
--- a/xgboost/src/main/java/hivemall/xgboost/XGBoostBaseUDTF.java
+++ /dev/null
@@ -1,337 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package hivemall.xgboost;
-
-import hivemall.UDTFWithOptions;
-import hivemall.utils.hadoop.HadoopUtils;
-import hivemall.utils.hadoop.HiveUtils;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.annotation.Nonnull;
-
-import ml.dmlc.xgboost4j.LabeledPoint;
-import ml.dmlc.xgboost4j.java.Booster;
-import ml.dmlc.xgboost4j.java.DMatrix;
-import ml.dmlc.xgboost4j.java.XGBoostError;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Options;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-
-/**
- * This is a base class to handle the options for XGBoost and provide common functions among various
- * tasks.
- */
-public abstract class XGBoostBaseUDTF extends UDTFWithOptions {
- private static final Log logger = LogFactory.getLog(XGBoostBaseUDTF.class);
-
- // Settings for the XGBoost native library
- static {
- NativeLibLoader.initXGBoost();
- }
-
- // For input buffer
- private final List<LabeledPoint> featuresList;
-
- // For input parameters
- private ListObjectInspector featureListOI;
- private PrimitiveObjectInspector featureElemOI;
- private PrimitiveObjectInspector targetOI;
-
- // For XGBoost options
- @Nonnull
- protected final Map<String, Object> params = new HashMap<String, Object>();
-
- // XGBoost options can be found in https://github.com/dmlc/xgboost/blob/master/doc/parameter.md
- // Most of default parameters are set along with the official one.
- {
- /** General parameters */
- params.put("booster", "gbtree");
- params.put("num_round", 8);
- params.put("silent", 1);
- // Set to 1 by default because most of distributed systems assume
- // each worker has a single vcore.
- params.put("nthread", 1);
-
- /** Parameters for both boosters */
- params.put("alpha", 0.0);
- // This default value depends on a booster type
- // params.put("lambda", 0.0);
-
- /** Parameters for Tree Booster */
- params.put("eta", 0.3);
- params.put("gamma", 0.0);
- params.put("max_depth", 6);
- params.put("min_child_weight", 1);
- params.put("max_delta_step", 0);
- params.put("subsample", 1.0);
- params.put("colsample_bytree", 1.0);
- params.put("colsample_bylevel", 1.0);
- // The memory-based version of XGBoost only supports `exact`
- params.put("tree_method", "exact");
-
- /** Learning Task Parameters */
- params.put("base_score", 0.5);
- }
-
- public XGBoostBaseUDTF() {
- this.featuresList = new ArrayList<>(1024);
- }
-
- @Override
- protected Options getOptions() {
- final Options opts = new Options();
-
- /** General parameters */
- opts.addOption("booster", true,
- "Set a booster to use, gbtree or gblinear. [default: gbree]");
- opts.addOption("num_round", true, "Number of boosting iterations [default: 8]");
- opts.addOption("silent", true,
- "0 means printing running messages, 1 means silent mode [default: 1]");
- opts.addOption("nthread", true,
- "Number of parallel threads used to run xgboost [default: 1]");
- opts.addOption("num_pbuffer", true,
- "Size of prediction buffer [set automatically by xgboost]");
- opts.addOption("num_feature", true,
- "Feature dimension used in boosting [default: set automatically by xgboost]");
-
- /** Parameters for both boosters */
- opts.addOption("alpha", true, "L1 regularization term on weights [default: 0.0]");
- opts.addOption("lambda", true,
- "L2 regularization term on weights [default: 1.0 for gbtree, 0.0 for gblinear]");
-
- /** Parameters for Tree Booster */
- opts.addOption("eta", true,
- "Step size shrinkage used in update to prevents overfitting [default: 0.3]");
- opts.addOption("gamma", true,
- "Minimum loss reduction required to make a further partition on a leaf node of the tree [default: 0.0]");
- opts.addOption("max_depth", true, "Max depth of decision tree [default: 6]");
- opts.addOption("min_child_weight", true,
- "Minimum sum of instance weight(hessian) needed in a child [default: 1]");
- opts.addOption("max_delta_step", true,
- "Maximum delta step we allow each tree's weight estimation to be [default: 0]");
- opts.addOption("subsample", true,
- "Subsample ratio of the training instance [default: 1.0]");
- opts.addOption("colsample_bytree", true,
- "Subsample ratio of columns when constructing each tree [default: 1.0]");
- opts.addOption("colsample_bylevel", true,
- "Subsample ratio of columns for each split, in each level [default: 1.0]");
-
- /** Parameters for Linear Booster */
- opts.addOption("lambda_bias", true, "L2 regularization term on bias [default: 0.0]");
-
- /** Learning Task Parameters */
- opts.addOption("base_score", true,
- "Initial prediction score of all instances, global bias [default: 0.5]");
- opts.addOption("eval_metric", true,
- "Evaluation metrics for validation data [default according to objective]");
-
- return opts;
- }
-
- @Override
- protected CommandLine processOptions(ObjectInspector[] argOIs) throws UDFArgumentException {
- CommandLine cl = null;
- if (argOIs.length >= 3) {
- final String rawArgs = HiveUtils.getConstString(argOIs[2]);
- cl = this.parseOptions(rawArgs);
-
- /** General parameters */
- if (cl.hasOption("booster")) {
- params.put("booster", cl.getOptionValue("booster"));
- }
- if (cl.hasOption("num_round")) {
- params.put("num_round", Integer.valueOf(cl.getOptionValue("num_round")));
- }
- if (cl.hasOption("silent")) {
- params.put("silent", Integer.valueOf(cl.getOptionValue("silent")));
- }
- if (cl.hasOption("nthread")) {
- params.put("nthread", Integer.valueOf(cl.getOptionValue("nthread")));
- }
- if (cl.hasOption("num_pbuffer")) {
- params.put("num_pbuffer", Integer.valueOf(cl.getOptionValue("num_pbuffer")));
- }
- if (cl.hasOption("num_feature")) {
- params.put("num_feature", Integer.valueOf(cl.getOptionValue("num_feature")));
- }
-
- /** Parameters for both boosters */
- if (cl.hasOption("alpha")) {
- params.put("alpha", Double.valueOf(cl.getOptionValue("alpha")));
- }
- if (cl.hasOption("lambda")) {
- params.put("lambda", Double.valueOf(cl.getOptionValue("lambda")));
- }
-
- /** Parameters for Tree Booster */
- if (cl.hasOption("eta")) {
- params.put("eta", Double.valueOf(cl.getOptionValue("eta")));
- }
- if (cl.hasOption("gamma")) {
- params.put("gamma", Double.valueOf(cl.getOptionValue("gamma")));
- }
- if (cl.hasOption("max_depth")) {
- params.put("max_depth", Integer.valueOf(cl.getOptionValue("max_depth")));
- }
- if (cl.hasOption("min_child_weight")) {
- params.put("min_child_weight",
- Integer.valueOf(cl.getOptionValue("min_child_weight")));
- }
- if (cl.hasOption("max_delta_step")) {
- params.put("max_delta_step", Integer.valueOf(cl.getOptionValue("max_delta_step")));
- }
- if (cl.hasOption("subsample")) {
- params.put("subsample", Double.valueOf(cl.getOptionValue("subsample")));
- }
- if (cl.hasOption("colsample_bytree")) {
- params.put("colsamle_bytree",
- Double.valueOf(cl.getOptionValue("colsample_bytree")));
- }
- if (cl.hasOption("colsample_bylevel")) {
- params.put("colsamle_bylevel",
- Double.valueOf(cl.getOptionValue("colsample_bylevel")));
- }
-
- /** Parameters for Linear Booster */
- if (cl.hasOption("lambda_bias")) {
- params.put("lambda_bias", Double.valueOf(cl.getOptionValue("lambda_bias")));
- }
-
- /** Learning Task Parameters */
- if (cl.hasOption("base_score")) {
- params.put("base_score", Double.valueOf(cl.getOptionValue("base_score")));
- }
- if (cl.hasOption("eval_metric")) {
- params.put("eval_metric", cl.getOptionValue("eval_metric"));
- }
- }
-
- try {
- // Try to create a `Booster` instance to check if given XGBoost options
- // are valid, or not.
- createXGBooster(params, featuresList);
- } catch (Exception e) {
- throw new UDFArgumentException(e);
- }
-
- return cl;
- }
-
- /** All the functions return (string model_id, byte[] pred_model) as built models */
- @Nonnull
- private static StructObjectInspector getReturnOIs() {
- final List<String> fieldNames = new ArrayList<>(2);
- final List<ObjectInspector> fieldOIs = new ArrayList<>(2);
- fieldNames.add("model_id");
- fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
- fieldNames.add("pred_model");
- fieldOIs.add(PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector);
- return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
- }
-
- @Override
- public StructObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
- throws UDFArgumentException {
- processOptions(argOIs);
- final ListObjectInspector listOI = HiveUtils.asListOI(argOIs[0]);
- final ObjectInspector elemOI = listOI.getListElementObjectInspector();
- this.featureListOI = listOI;
- this.featureElemOI = HiveUtils.asStringOI(elemOI);
- this.targetOI = HiveUtils.asDoubleCompatibleOI(argOIs[1]);
- return getReturnOIs();
- }
-
- /** It `target` has valid input range, it overrides this */
- protected void checkTargetValue(double target) throws HiveException {}
-
- @Override
- public void process(@Nonnull Object[] args) throws HiveException {
- if (args[0] == null) {
- return;
- }
-
- // TODO: Need to support dense inputs
- final List<?> features = (List<?>) featureListOI.getList(args[0]);
- final String[] fv = new String[features.size()];
- for (int i = 0; i < features.size(); i++) {
- fv[i] = (String) featureElemOI.getPrimitiveJavaObject(features.get(i));
- }
- double target = PrimitiveObjectInspectorUtils.getDouble(args[1], this.targetOI);
- checkTargetValue(target);
- final LabeledPoint point = XGBoostUtils.parseFeatures(target, fv);
- if (point != null) {
- this.featuresList.add(point);
- }
- }
-
- @Nonnull
- private static String generateUniqueModelId() {
- return "xgbmodel-" + HadoopUtils.getUniqueTaskIdString();
- }
-
- @Nonnull
- private static Booster createXGBooster(final Map<String, Object> params,
- final List<LabeledPoint> input) throws NoSuchMethodException, XGBoostError,
- IllegalAccessException, InvocationTargetException, InstantiationException {
- Class<?>[] args = {Map.class, DMatrix[].class};
- Constructor<Booster> ctor = Booster.class.getDeclaredConstructor(args);
- ctor.setAccessible(true);
- return ctor.newInstance(
- new Object[] {params, new DMatrix[] {new DMatrix(input.iterator(), "")}});
- }
-
- @Override
- public void close() throws HiveException {
- try {
- // Kick off training with XGBoost
- final DMatrix trainData = new DMatrix(featuresList.iterator(), "");
- final Booster booster = createXGBooster(params, featuresList);
- final int num_round = (Integer) params.get("num_round");
- for (int i = 0; i < num_round; i++) {
- booster.update(trainData, i);
- }
-
- // Output the built model
- final String modelId = generateUniqueModelId();
- final byte[] predModel = booster.toByteArray();
- logger.info("model_id:" + modelId.toString() + " size:" + predModel.length);
- forward(new Object[] {modelId, predModel});
- } catch (Exception e) {
- throw new HiveException(e);
- }
- }
-
-}
diff --git a/xgboost/src/main/java/hivemall/xgboost/XGBoostBatchPredictUDTF.java b/xgboost/src/main/java/hivemall/xgboost/XGBoostBatchPredictUDTF.java
new file mode 100644
index 0000000..dd35693
--- /dev/null
+++ b/xgboost/src/main/java/hivemall/xgboost/XGBoostBatchPredictUDTF.java
@@ -0,0 +1,350 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.xgboost;
+
+import hivemall.UDTFWithOptions;
+import hivemall.utils.collections.lists.FloatArrayList;
+import hivemall.utils.collections.lists.IntArrayList;
+import hivemall.utils.hadoop.HiveUtils;
+import hivemall.utils.hadoop.WritableUtils;
+import hivemall.utils.lang.Primitives;
+import hivemall.xgboost.utils.NativeLibLoader;
+import hivemall.xgboost.utils.XGBoostUtils;
+import ml.dmlc.xgboost4j.LabeledPoint;
+import ml.dmlc.xgboost4j.java.Booster;
+import ml.dmlc.xgboost4j.java.DMatrix;
+import ml.dmlc.xgboost4j.java.XGBoostError;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+//@formatter:off
+@Description(name = "xgboost_batch_predict",
+ value = "_FUNC_(PRIMITIVE rowid, array<string|double> features, string model_id, array<string> pred_model [, string options]) "
+ + "- Returns a prediction result as (string rowid, array<double> predicted)",
+ extended = "select\n" +
+ " rowid, \n" +
+ " array_avg(predicted) as predicted,\n" +
+ " avg(predicted[0]) as predicted0\n" +
+ "from (\n" +
+ " select\n" +
+ " xgboost_batch_predict(rowid, features, model_id, model) as (rowid, predicted)\n" +
+ " from\n" +
+ " xgb_model l\n" +
+ " LEFT OUTER JOIN xgb_input r\n" +
+ ") t\n" +
+ "group by rowid;")
+//@formatter:on
+public final class XGBoostBatchPredictUDTF extends UDTFWithOptions {
+
+ // For input parameters
+ private PrimitiveObjectInspector rowIdOI;
+ private ListObjectInspector featureListOI;
+ private boolean denseFeatures;
+ @Nullable
+ private PrimitiveObjectInspector featureElemOI;
+ private StringObjectInspector modelIdOI;
+ private StringObjectInspector modelOI;
+
+ // For input buffer
+ private transient Map<String, Booster> mapToModel;
+ private transient Map<String, List<LabeledPointWithRowId>> rowBuffer;
+
+ private int _batchSize;
+
+ @Nonnull
+ protected transient final Object[] _forwardObj;
+
+ // Settings for the XGBoost native library
+ static {
+ NativeLibLoader.initXGBoost();
+ }
+
+ public XGBoostBatchPredictUDTF() {
+ super();
+ this._forwardObj = new Object[2];
+ }
+
+ @Override
+ protected Options getOptions() {
+ Options opts = new Options();
+ opts.addOption("batch_size", true, "Number of rows to predict together [default: 128]");
+ return opts;
+ }
+
+ @Override
+ protected CommandLine processOptions(ObjectInspector[] argOIs) throws UDFArgumentException {
+ int batchSize = 128;
+ CommandLine cl = null;
+ if (argOIs.length >= 5) {
+ String rawArgs = HiveUtils.getConstString(argOIs, 4);
+ cl = parseOptions(rawArgs);
+ batchSize = Primitives.parseInt(cl.getOptionValue("batch_size"), batchSize);
+ if (batchSize < 1) {
+ throw new UDFArgumentException("batch_size must be greater than 0: " + batchSize);
+ }
+ }
+ this._batchSize = batchSize;
+ return cl;
+ }
+
+ @Override
+ public StructObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
+ throws UDFArgumentException {
+ if (argOIs.length != 4 && argOIs.length != 5) {
+ showHelp("Invalid argment length=" + argOIs.length);
+ }
+ processOptions(argOIs);
+
+ this.rowIdOI = HiveUtils.asPrimitiveObjectInspector(argOIs, 0);
+
+ this.featureListOI = HiveUtils.asListOI(argOIs, 1);
+ ObjectInspector elemOI = featureListOI.getListElementObjectInspector();
+ if (HiveUtils.isNumberOI(elemOI)) {
+ this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
+ this.denseFeatures = true;
+ } else if (HiveUtils.isStringOI(elemOI)) {
+ this.denseFeatures = false;
+ } else {
+ throw new UDFArgumentException(
+ "Expected array<string|double> for the 2nd argment but got an unexpected features type: "
+ + featureListOI.getTypeName());
+ }
+ this.modelIdOI = HiveUtils.asStringOI(argOIs, 2);
+ this.modelOI = HiveUtils.asStringOI(argOIs, 3);
+
+ return getReturnOI(rowIdOI);
+ }
+
+ /** Override this to output predicted results depending on a task type */
+ /** Return (string rowid, array<double> predicted) as a result */
+ @Nonnull
+ protected StructObjectInspector getReturnOI(@Nonnull PrimitiveObjectInspector rowIdOI) {
+ List<String> fieldNames = new ArrayList<>(2);
+ List<ObjectInspector> fieldOIs = new ArrayList<>(2);
+ fieldNames.add("rowid");
+ fieldOIs.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
+ rowIdOI.getPrimitiveCategory()));
+ fieldNames.add("predicted");
+ fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.writableFloatObjectInspector));
+ return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
+ }
+
+ @Override
+ public void process(Object[] args) throws HiveException {
+ if (mapToModel == null) {
+ this.mapToModel = new HashMap<String, Booster>();
+ this.rowBuffer = new HashMap<String, List<LabeledPointWithRowId>>();
+ }
+ if (args[1] == null) {
+ return;
+ }
+
+ String modelId =
+ PrimitiveObjectInspectorUtils.getString(nonNullArgument(args, 2), modelIdOI);
+ Booster model = mapToModel.get(modelId);
+ if (model == null) {
+ Text arg3 = modelOI.getPrimitiveWritableObject(nonNullArgument(args, 3));
+ model = XGBoostUtils.deserializeBooster(arg3);
+ mapToModel.put(modelId, model);
+ }
+
+ List<LabeledPointWithRowId> rowBatch = rowBuffer.get(modelId);
+ if (rowBatch == null) {
+ rowBatch = new ArrayList<LabeledPointWithRowId>(_batchSize);
+ rowBuffer.put(modelId, rowBatch);
+ }
+ LabeledPointWithRowId row = parseRow(args);
+ rowBatch.add(row);
+ if (rowBatch.size() >= _batchSize) {
+ predictAndFlush(model, rowBatch);
+ }
+ }
+
+ @Nonnull
+ private LabeledPointWithRowId parseRow(@Nonnull Object[] args) throws UDFArgumentException {
+ final Writable rowId = HiveUtils.copyToWritable(nonNullArgument(args, 0), rowIdOI);
+
+ final Object arg1 = args[1];
+ if (denseFeatures) {
+ return parseDenseFeatures(rowId, arg1, featureListOI, featureElemOI);
+ } else {
+ return parseSparseFeatures(rowId, arg1, featureListOI);
+ }
+ }
+
+ @Nonnull
+ private static LabeledPointWithRowId parseDenseFeatures(@Nonnull final Writable rowId,
+ @Nonnull final Object argObj, @Nonnull final ListObjectInspector featureListOI,
+ @Nonnull final PrimitiveObjectInspector featureElemOI) throws UDFArgumentException {
+ final int size = featureListOI.getListLength(argObj);
+
+ final float[] values = new float[size];
+ for (int i = 0; i < size; i++) {
+ final Object o = featureListOI.getListElement(argObj, i);
+ if (o == null) {
+ values[i] = Float.NaN;
+ } else {
+ float v = PrimitiveObjectInspectorUtils.getFloat(o, featureElemOI);
+ values[i] = v;
+ }
+ }
+
+ return new LabeledPointWithRowId(rowId, /* dummy label */ 0.f, null, values);
+
+ }
+
+ @Nonnull
+ private static LabeledPointWithRowId parseSparseFeatures(@Nonnull final Writable rowId,
+ @Nonnull final Object argObj, @Nonnull final ListObjectInspector featureListOI)
+ throws UDFArgumentException {
+ final int size = featureListOI.getListLength(argObj);
+ final IntArrayList indices = new IntArrayList(size);
+ final FloatArrayList values = new FloatArrayList(size);
+
+ for (int i = 0; i < size; i++) {
+ Object f = featureListOI.getListElement(argObj, i);
+ if (f == null) {
+ continue;
+ }
+ final String str = f.toString();
+ final int pos = str.indexOf(':');
+ if (pos < 1) {
+ throw new UDFArgumentException("Invalid feature format: " + str);
+ }
+ final int index;
+ final float value;
+ try {
+ index = Integer.parseInt(str.substring(0, pos));
+ value = Float.parseFloat(str.substring(pos + 1));
+ } catch (NumberFormatException e) {
+ throw new UDFArgumentException("Failed to parse a feature value: " + str);
+ }
+ indices.add(index);
+ values.add(value);
+ }
+
+ return new LabeledPointWithRowId(rowId, /* dummy label */ 0.f, indices.toArray(),
+ values.toArray());
+ }
+
+
+ @Override
+ public void close() throws HiveException {
+ for (Entry<String, List<LabeledPointWithRowId>> e : rowBuffer.entrySet()) {
+ String modelId = e.getKey();
+ List<LabeledPointWithRowId> rowBatch = e.getValue();
+ if (rowBatch.isEmpty()) {
+ continue;
+ }
+ final Booster model = Objects.requireNonNull(mapToModel.get(modelId));
+ try {
+ predictAndFlush(model, rowBatch);
+ } finally {
+ XGBoostUtils.close(model);
+ }
+ }
+ this.rowBuffer = null;
+ this.mapToModel = null;
+ }
+
+ private void predictAndFlush(@Nonnull final Booster model,
+ @Nonnull final List<LabeledPointWithRowId> rowBatch) throws HiveException {
+ DMatrix testData = null;
+ final float[][] predicted;
+ try {
+ testData = XGBoostUtils.createDMatrix(rowBatch);
+ predicted = model.predict(testData);
+ } catch (XGBoostError e) {
+ throw new HiveException("Exception caused at prediction", e);
+ } finally {
+ XGBoostUtils.close(testData);
+ }
+ forwardPredicted(rowBatch, predicted);
+ rowBatch.clear();
+ }
+
+ private void forwardPredicted(@Nonnull final List<LabeledPointWithRowId> rowBatch,
+ @Nonnull final float[][] predicted) throws HiveException {
+ if (rowBatch.size() != predicted.length) {
+ throw new HiveException(String.format("buf.size() = %d but predicted.length = %d",
+ rowBatch.size(), predicted.length));
+ }
+ if (predicted.length == 0) {
+ return;
+ }
+
+ final int ncols = predicted[0].length;
+ final List<FloatWritable> list = WritableUtils.newFloatList(ncols);
+
+ final Object[] forwardObj = this._forwardObj;
+ forwardObj[1] = list;
+
+ for (int i = 0; i < predicted.length; i++) {
+ Writable rowId = Objects.requireNonNull(rowBatch.get(i)).getRowId();
+ forwardObj[0] = rowId;
+ WritableUtils.setValues(predicted[i], list);
+ forward(forwardObj);
+ }
+ }
+
+ public static final class LabeledPointWithRowId extends LabeledPoint {
+ private static final long serialVersionUID = -7150841669515184648L;
+
+ @Nonnull
+ final Writable rowId;
+
+ LabeledPointWithRowId(@Nonnull Writable rowId, float label, @Nullable int[] indices,
+ @Nonnull float[] values) {
+ super(label, indices, values);
+ this.rowId = rowId;
+ }
+
+ @Nonnull
+ public Writable getRowId() {
+ return rowId;
+ }
+ }
+
+}
diff --git a/xgboost/src/main/java/hivemall/xgboost/XGBoostOnlinePredictUDTF.java b/xgboost/src/main/java/hivemall/xgboost/XGBoostOnlinePredictUDTF.java
new file mode 100644
index 0000000..ee3f2b6
--- /dev/null
+++ b/xgboost/src/main/java/hivemall/xgboost/XGBoostOnlinePredictUDTF.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.xgboost;
+
+import biz.k11i.xgboost.Predictor;
+import biz.k11i.xgboost.util.FVec;
+import hivemall.UDTFWithOptions;
+import hivemall.utils.hadoop.HiveUtils;
+import hivemall.utils.hadoop.WritableUtils;
+import hivemall.xgboost.utils.XGBoostUtils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+//@formatter:off
+@Description(name = "xgboost_predict",
+ value = "_FUNC_(PRIMITIVE rowid, array<string|double> features, string model_id, array<string> pred_model [, string options]) "
+ + "- Returns a prediction result as (string rowid, array<double> predicted)",
+ extended = "select\n" +
+ " rowid, \n" +
+ " array_avg(predicted) as predicted,\n" +
+ " avg(predicted[0]) as predicted0\n" +
+ "from (\n" +
+ " select\n" +
+ " xgboost_predict(rowid, features, model_id, model) as (rowid, predicted)\n" +
+ " from\n" +
+ " xgb_model l\n" +
+ " LEFT OUTER JOIN xgb_input r\n" +
+ ") t\n" +
+ "group by rowid;")
+//@formatter:on
+public class XGBoostOnlinePredictUDTF extends UDTFWithOptions {
+
+ // For input parameters
+ private PrimitiveObjectInspector rowIdOI;
+ private ListObjectInspector featureListOI;
+ private boolean denseFeatures;
+ @Nullable
+ private PrimitiveObjectInspector featureElemOI;
+ private StringObjectInspector modelIdOI;
+ private StringObjectInspector modelOI;
+
+ // For input buffer
+ @Nullable
+ private transient Map<String, Predictor> mapToModel;
+
+ @Nonnull
+ protected transient final Object[] _forwardObj;
+ @Nullable
+ protected transient List<DoubleWritable> _predictedCache;
+
+ public XGBoostOnlinePredictUDTF() {
+ this(new Object[2]);
+ }
+
+ protected XGBoostOnlinePredictUDTF(@Nonnull Object[] forwardObj) {
+ super();
+ this._forwardObj = forwardObj;
+ }
+
+ @Override
+ protected Options getOptions() {
+ Options opts = new Options();
+ // not yet supported
+ return opts;
+ }
+
+ @Override
+ protected CommandLine processOptions(ObjectInspector[] argOIs) throws UDFArgumentException {
+ CommandLine cl = null;
+ if (argOIs.length >= 5) {
+ String rawArgs = HiveUtils.getConstString(argOIs, 4);
+ cl = parseOptions(rawArgs);
+ }
+ return cl;
+ }
+
+ @Override
+ public StructObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
+ throws UDFArgumentException {
+ if (argOIs.length != 4 && argOIs.length != 5) {
+ showHelp("Invalid argment length=" + argOIs.length);
+ }
+ processOptions(argOIs);
+
+ this.rowIdOI = HiveUtils.asPrimitiveObjectInspector(argOIs, 0);
+ ListObjectInspector listOI = HiveUtils.asListOI(argOIs, 1);
+ this.featureListOI = listOI;
+ ObjectInspector elemOI = listOI.getListElementObjectInspector();
+ if (HiveUtils.isNumberOI(elemOI)) {
+ this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
+ this.denseFeatures = true;
+ } else if (HiveUtils.isStringOI(elemOI)) {
+ this.denseFeatures = false;
+ } else {
+ throw new UDFArgumentException(
+ "Expected array<string|double> for the 2nd argment but got an unexpected features type: "
+ + listOI.getTypeName());
+ }
+ this.modelIdOI = HiveUtils.asStringOI(argOIs, 2);
+ this.modelOI = HiveUtils.asStringOI(argOIs, 3);
+ return getReturnOI(rowIdOI);
+ }
+
+ /** Override this to output predicted results depending on a task type */
+ /** Return (primitive rowid, array<double> predicted) as a result */
+ @Nonnull
+ protected StructObjectInspector getReturnOI(@Nonnull PrimitiveObjectInspector rowIdOI) {
+ List<String> fieldNames = new ArrayList<>(2);
+ List<ObjectInspector> fieldOIs = new ArrayList<>(2);
+ fieldNames.add("rowid");
+ fieldOIs.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
+ rowIdOI.getPrimitiveCategory()));
+ fieldNames.add("predicted");
+ fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
+ return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
+ }
+
+ @Override
+ public void process(Object[] args) throws HiveException {
+ if (mapToModel == null) {
+ this.mapToModel = new HashMap<String, Predictor>();
+ }
+ if (args[1] == null) {// features is null
+ return;
+ }
+
+ String modelId =
+ PrimitiveObjectInspectorUtils.getString(nonNullArgument(args, 2), modelIdOI);
+ Predictor model = mapToModel.get(modelId);
+ if (model == null) {
+ Text arg3 = modelOI.getPrimitiveWritableObject(nonNullArgument(args, 3));
+ model = XGBoostUtils.loadPredictor(arg3);
+ mapToModel.put(modelId, model);
+ }
+
+ Writable rowId = HiveUtils.copyToWritable(nonNullArgument(args, 0), rowIdOI);
+ FVec features = denseFeatures ? parseDenseFeatures(args[1])
+ : parseSparseFeatures(featureListOI.getList(args[1]));
+
+ predictAndForward(model, rowId, features);
+ }
+
+ @Nonnull
+ private FVec parseDenseFeatures(@Nonnull Object argObj) throws UDFArgumentException {
+ final int length = featureListOI.getListLength(argObj);
+ final double[] values = new double[length];
+ for (int i = 0; i < length; i++) {
+ final Object o = featureListOI.getListElement(argObj, i);
+ final double v;
+ if (o == null) {
+ v = Double.NaN;
+ } else {
+ v = PrimitiveObjectInspectorUtils.getDouble(o, featureElemOI);
+ }
+ values[i] = v;
+
+ }
+ return FVec.Transformer.fromArray(values, false);
+ }
+
+ @Nonnull
+ private static FVec parseSparseFeatures(@Nonnull final List<?> featureList)
+ throws UDFArgumentException {
+ final Map<Integer, Double> map = new HashMap<>((int) (featureList.size() * 1.5));
+ for (Object f : featureList) {
+ if (f == null) {
+ continue;
+ }
+ String str = f.toString();
+ final int pos = str.indexOf(':');
+ if (pos < 1) {
+ throw new UDFArgumentException("Invalid feature format: " + str);
+ }
+ final int index;
+ final double value;
+ try {
+ index = Integer.parseInt(str.substring(0, pos));
+ value = Double.parseDouble(str.substring(pos + 1));
+ } catch (NumberFormatException e) {
+ throw new UDFArgumentException("Failed to parse a feature value: " + str);
+ }
+ map.put(index, value);
+ }
+
+ return FVec.Transformer.fromMap(map);
+ }
+
+ private void predictAndForward(@Nonnull final Predictor model, @Nonnull final Writable rowId,
+ @Nonnull final FVec features) throws HiveException {
+ double[] predicted = model.predict(features);
+ // predicted[0] has
+ // - probability ("binary:logistic")
+ // - class label ("multi:softmax")
+ forwardPredicted(rowId, predicted);
+ }
+
+ protected void forwardPredicted(@Nonnull final Writable rowId,
+ @Nonnull final double[] predicted) throws HiveException {
+ List<DoubleWritable> list = WritableUtils.toWritableList(predicted, _predictedCache);
+ this._predictedCache = list;
+ Object[] forwardObj = this._forwardObj;
+ forwardObj[0] = rowId;
+ forwardObj[1] = list;
+ forward(forwardObj);
+ }
+
+ @Override
+ public void close() throws HiveException {
+ this.mapToModel = null;
+ }
+
+}
diff --git a/xgboost/src/main/java/hivemall/xgboost/XGBoostPredictOneUDTF.java b/xgboost/src/main/java/hivemall/xgboost/XGBoostPredictOneUDTF.java
new file mode 100644
index 0000000..80cdddb
--- /dev/null
+++ b/xgboost/src/main/java/hivemall/xgboost/XGBoostPredictOneUDTF.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.xgboost;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.Nonnull;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.Writable;
+
+//@formatter:off
+@Description(name = "xgboost_predict_one",
+ value = "_FUNC_(PRIMITIVE rowid, array<string|double> features, string model_id, array<string> pred_model [, string options]) "
+ + "- Returns a prediction result as (string rowid, double predicted)",
+ extended = "select\n" +
+ " rowid, \n" +
+ " avg(predicted) as predicted\n" +
+ "from (\n" +
+ " select\n" +
+ " xgboost_predict_one(rowid, features, model_id, model) as (rowid, predicted)\n" +
+ " from\n" +
+ " xgb_model l\n" +
+ " LEFT OUTER JOIN xgb_input r\n" +
+ ") t\n" +
+ "group by rowid;")
+//@formatter:on
+public final class XGBoostPredictOneUDTF extends XGBoostOnlinePredictUDTF {
+
+ public XGBoostPredictOneUDTF() {
+ super(new Object[2]);
+ }
+
+ /** Return (string rowid, double predicted) as a result */
+ @Override
+ protected StructObjectInspector getReturnOI(@Nonnull PrimitiveObjectInspector rowIdOI) {
+ List<String> fieldNames = new ArrayList<>(2);
+ List<ObjectInspector> fieldOIs = new ArrayList<>(2);
+ fieldNames.add("rowid");
+ fieldOIs.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
+ rowIdOI.getPrimitiveCategory()));
+ fieldNames.add("predicted");
+ fieldOIs.add(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
+ return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
+ }
+
+ @Override
+ protected void forwardPredicted(@Nonnull Writable rowId, @Nonnull double[] predicted)
+ throws HiveException {
+ final Object[] forwardObj = _forwardObj;
+ forwardObj[0] = rowId;
+ forwardObj[1] = Double.valueOf(predicted[0]);
+ forward(forwardObj);
+ }
+
+}
diff --git a/xgboost/src/main/java/hivemall/xgboost/XGBoostPredictTripleUDTF.java b/xgboost/src/main/java/hivemall/xgboost/XGBoostPredictTripleUDTF.java
new file mode 100644
index 0000000..ce47dee
--- /dev/null
+++ b/xgboost/src/main/java/hivemall/xgboost/XGBoostPredictTripleUDTF.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.xgboost;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.Nonnull;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.Writable;
+
+//@formatter:off
+@Description(name = "xgboost_predict_triple",
+ value = "_FUNC_(PRIMITIVE rowid, array<string|double> features, string model_id, array<string> pred_model [, string options]) "
+ + "- Returns a prediction result as (string rowid, string label, double probability)",
+ extended = "select\n" +
+ " rowid,\n" +
+ " label,\n" +
+ " avg(prob) as prob\n" +
+ "from (\n" +
+ " select\n" +
+ " xgboost_predict_triple(rowid, features, model_id, model) as (rowid, label, prob)\n" +
+ " from\n" +
+ " xgb_model l\n" +
+ " LEFT OUTER JOIN xgb_input r\n" +
+ ") t\n" +
+ "group by rowid, label;")
+//@formatter:on
+public final class XGBoostPredictTripleUDTF extends XGBoostOnlinePredictUDTF {
+
+ public XGBoostPredictTripleUDTF() {
+ super(new Object[3]);
+ }
+
+ /** Return (string rowid, int label, double probability) as a result */
+ @Override
+ protected StructObjectInspector getReturnOI(@Nonnull PrimitiveObjectInspector rowIdOI) {
+ List<String> fieldNames = new ArrayList<>(3);
+ List<ObjectInspector> fieldOIs = new ArrayList<>(3);
+ fieldNames.add("rowid");
+ fieldOIs.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
+ rowIdOI.getPrimitiveCategory()));
+ fieldNames.add("label");
+ fieldOIs.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
+ fieldNames.add("proba");
+ fieldOIs.add(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
+
+ return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
+ }
+
+ @Override
+ protected void forwardPredicted(@Nonnull Writable rowId, @Nonnull double[] predicted)
+ throws HiveException {
+ final Object[] forwardObj = _forwardObj;
+ forwardObj[0] = rowId;
+ for (int j = 0, ncols = predicted.length; j < ncols; j++) {
+ forwardObj[1] = Integer.valueOf(j);
+ forwardObj[2] = Double.valueOf(predicted[j]);
+ forward(forwardObj);
+ }
+ }
+
+}
diff --git a/xgboost/src/main/java/hivemall/xgboost/XGBoostPredictUDTF.java b/xgboost/src/main/java/hivemall/xgboost/XGBoostPredictUDTF.java
deleted file mode 100644
index 9c3b476..0000000
--- a/xgboost/src/main/java/hivemall/xgboost/XGBoostPredictUDTF.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package hivemall.xgboost;
-
-import hivemall.UDTFWithOptions;
-import hivemall.utils.hadoop.HiveUtils;
-import hivemall.utils.lang.Primitives;
-
-import java.io.ByteArrayInputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import javax.annotation.Nonnull;
-
-import ml.dmlc.xgboost4j.LabeledPoint;
-import ml.dmlc.xgboost4j.java.Booster;
-import ml.dmlc.xgboost4j.java.DMatrix;
-import ml.dmlc.xgboost4j.java.XGBoost;
-import ml.dmlc.xgboost4j.java.XGBoostError;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-
-public abstract class XGBoostPredictUDTF extends UDTFWithOptions {
-
- // For input parameters
- private PrimitiveObjectInspector rowIdOI;
- private ListObjectInspector featureListOI;
- private PrimitiveObjectInspector featureElemOI;
- private PrimitiveObjectInspector modelIdOI;
- private PrimitiveObjectInspector modelOI;
-
- // For input buffer
- private Map<String, Booster> mapToModel;
- private Map<String, List<LabeledPointWithRowId>> rowBuffer;
-
- private int batch_size;
-
- // Settings for the XGBoost native library
- static {
- NativeLibLoader.initXGBoost();
- }
-
- public XGBoostPredictUDTF() {
- super();
- }
-
- @Override
- protected Options getOptions() {
- Options opts = new Options();
- opts.addOption("batch_size", true, "Number of rows to predict together [default: 128]");
- return opts;
- }
-
- @Override
- protected CommandLine processOptions(ObjectInspector[] argOIs) throws UDFArgumentException {
- int _batch_size = 128;
- CommandLine cl = null;
- if (argOIs.length >= 5) {
- String rawArgs = HiveUtils.getConstString(argOIs[4]);
- cl = this.parseOptions(rawArgs);
- _batch_size = Primitives.parseInt(cl.getOptionValue("_batch_size"), _batch_size);
- if (_batch_size < 1) {
- throw new IllegalArgumentException(
- "batch_size must be greater than 0: " + _batch_size);
- }
- }
- this.batch_size = _batch_size;
- return cl;
- }
-
- /** Override this to output predicted results depending on a task type */
- @Nonnull
- protected abstract StructObjectInspector getReturnOI();
-
- protected abstract void forwardPredicted(@Nonnull final List<LabeledPointWithRowId> testData,
- @Nonnull final float[][] predicted) throws HiveException;
-
- @Override
- public StructObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
- throws UDFArgumentException {
- if (argOIs.length != 4 && argOIs.length != 5) {
- throw new UDFArgumentException(this.getClass().getSimpleName()
- + " takes 4 or 5 arguments: string rowid, string[] features, string model_id,"
- + " array<byte> pred_model [, string options]: " + argOIs.length);
- } else {
- this.processOptions(argOIs);
- this.rowIdOI = HiveUtils.asStringOI(argOIs[0]);
- final ListObjectInspector listOI = HiveUtils.asListOI(argOIs[1]);
- final ObjectInspector elemOI = listOI.getListElementObjectInspector();
- this.featureListOI = listOI;
- this.featureElemOI = HiveUtils.asStringOI(elemOI);
- this.modelIdOI = HiveUtils.asStringOI(argOIs[2]);
- this.modelOI = HiveUtils.asBinaryOI(argOIs[3]);
- this.mapToModel = new HashMap<String, Booster>();
- this.rowBuffer = new HashMap<String, List<LabeledPointWithRowId>>();
- return getReturnOI();
- }
- }
-
- @Nonnull
- private static DMatrix createDMatrix(@Nonnull final List<LabeledPointWithRowId> data)
- throws XGBoostError {
- final List<LabeledPoint> points = new ArrayList<>(data.size());
- for (LabeledPointWithRowId d : data) {
- points.add(d.point);
- }
- return new DMatrix(points.iterator(), "");
- }
-
- @Nonnull
- private static Booster initXgBooster(@Nonnull final byte[] input) throws HiveException {
- try {
- return XGBoost.loadModel(new ByteArrayInputStream(input));
- } catch (Exception e) {
- throw new HiveException(e);
- }
- }
-
- private void predictAndFlush(final Booster model, final List<LabeledPointWithRowId> buf)
- throws HiveException {
- final DMatrix testData;
- final float[][] predicted;
- try {
- testData = createDMatrix(buf);
- predicted = model.predict(testData);
- } catch (XGBoostError e) {
- throw new HiveException(e);
- }
- forwardPredicted(buf, predicted);
- buf.clear();
- }
-
- @Override
- public void process(Object[] args) throws HiveException {
- if (args[1] == null) {
- return;
- }
-
- final String rowId = PrimitiveObjectInspectorUtils.getString(args[0], rowIdOI);
- final List<?> features = (List<?>) featureListOI.getList(args[1]);
- final String[] fv = new String[features.size()];
- for (int i = 0; i < features.size(); i++) {
- fv[i] = (String) featureElemOI.getPrimitiveJavaObject(features.get(i));
- }
- final String modelId = PrimitiveObjectInspectorUtils.getString(args[2], modelIdOI);
- if (!mapToModel.containsKey(modelId)) {
- final byte[] predModel =
- PrimitiveObjectInspectorUtils.getBinary(args[3], modelOI).getBytes();
- mapToModel.put(modelId, initXgBooster(predModel));
- }
-
- final LabeledPoint point = XGBoostUtils.parseFeatures(0.f, fv);
- if (point == null) {
- return;
- }
-
- List<LabeledPointWithRowId> buf = rowBuffer.get(modelId);
- if (buf == null) {
- buf = new ArrayList<LabeledPointWithRowId>();
- rowBuffer.put(modelId, buf);
- }
- buf.add(new LabeledPointWithRowId(rowId, point));
- if (buf.size() >= batch_size) {
- predictAndFlush(mapToModel.get(modelId), buf);
- }
- }
-
- public static final class LabeledPointWithRowId {
-
- @Nonnull
- final String rowId;
- @Nonnull
- final LabeledPoint point;
-
- LabeledPointWithRowId(@Nonnull String rowId, @Nonnull LabeledPoint point) {
- this.rowId = rowId;
- this.point = point;
- }
-
- @Nonnull
- public String getRowId() {
- return rowId;
- }
-
- @Nonnull
- public LabeledPoint getPoint() {
- return point;
- }
- }
-
- @Override
- public void close() throws HiveException {
- for (Entry<String, List<LabeledPointWithRowId>> e : rowBuffer.entrySet()) {
- predictAndFlush(mapToModel.get(e.getKey()), e.getValue());
- }
- }
-
-}
diff --git a/xgboost/src/main/java/hivemall/xgboost/XGBoostTrainUDTF.java b/xgboost/src/main/java/hivemall/xgboost/XGBoostTrainUDTF.java
new file mode 100644
index 0000000..6922aca
--- /dev/null
+++ b/xgboost/src/main/java/hivemall/xgboost/XGBoostTrainUDTF.java
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.xgboost;
+
+import hivemall.UDTFWithOptions;
+import hivemall.annotations.VisibleForTesting;
+import hivemall.utils.collections.lists.FloatArrayList;
+import hivemall.utils.hadoop.HadoopUtils;
+import hivemall.utils.hadoop.HiveUtils;
+import hivemall.utils.lang.OptionUtils;
+import hivemall.utils.math.MathUtils;
+import hivemall.xgboost.utils.DMatrixBuilder;
+import hivemall.xgboost.utils.DenseDMatrixBuilder;
+import hivemall.xgboost.utils.NativeLibLoader;
+import hivemall.xgboost.utils.SparseDMatrixBuilder;
+import hivemall.xgboost.utils.XGBoostUtils;
+import matrix4j.utils.lang.ArrayUtils;
+import matrix4j.utils.lang.Primitives;
+import ml.dmlc.xgboost4j.java.Booster;
+import ml.dmlc.xgboost4j.java.DMatrix;
+import ml.dmlc.xgboost4j.java.XGBoostError;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import javax.annotation.Nonnegative;
+import javax.annotation.Nonnull;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.io.Text;
+
+/**
+ * UDTF for train_xgboost
+ */
+//@formatter:off
+@Description(name = "train_xgboost",
+ value = "_FUNC_(array<string|double> features, <int|double> target, const string options)"
+ + " - Returns a relation consists of <string model_id, array<string> pred_model>",
+ extended = "SELECT \n" +
+ " train_xgboost(features, label, '-objective binary:logistic -iters 10') \n" +
+ " as (model_id, model)\n" +
+ "from (\n" +
+ " select features, label\n" +
+ " from xgb_input\n" +
+ " cluster by rand(43) -- shuffle\n" +
+ ") shuffled;")
+//@formatter:on
+public class XGBoostTrainUDTF extends UDTFWithOptions {
+ private static final Log logger = LogFactory.getLog(XGBoostTrainUDTF.class);
+
+ // Settings for the XGBoost native library
+ static {
+ NativeLibLoader.initXGBoost();
+ }
+
+ // For input parameters
+ private ListObjectInspector featureListOI;
+ private PrimitiveObjectInspector featureElemOI;
+ private PrimitiveObjectInspector targetOI;
+
+ // For training input buffering
+ private boolean denseInput;
+ private DMatrixBuilder matrixBuilder;
+ private FloatArrayList labels;
+
+ // For XGBoost options
+ @Nonnull
+ protected final Map<String, Object> params = new HashMap<String, Object>();
+
+ protected int numClass;
+ protected ObjectiveType objectiveType = null;
+
+ public enum ObjectiveType {
+ regression, binary, multiclass, rank, other;
+
+ @Nonnull
+ public static ObjectiveType resolve(@Nonnull String objective) {
+ if (objective.startsWith("reg:")) {
+ return regression;
+ } else if (objective.startsWith("binary:")) {
+ return binary;
+ } else if (objective.startsWith("multi:")) {
+ return multiclass;
+ } else if (objective.startsWith("rank:")) {
+ return rank;
+ } else {
+ return other;
+ }
+ }
+ }
+
+
+ public XGBoostTrainUDTF() {}
+
+ @Override
+ protected Options getOptions() {
+ final Options opts = new Options();
+
+ opts.addOption("num_round", "iters", true, "Number of boosting iterations [default: 10]");
+ opts.addOption("maximize_evaluation_metrics", true,
+ "Maximize evaluation metrics [default: false]");
+ opts.addOption("num_early_stopping_rounds", true,
+ "Minimum rounds required for early stopping [default: 0]");
+ opts.addOption("validation_ratio", true,
+ "Validation ratio in range [0.0,1.0] [default: 0.2]");
+
+ /** General parameters */
+ opts.addOption("booster", true,
+ "Set a booster to use, gbtree or gblinear or dart. [default: gbree]");
+ opts.addOption("silent", true, "Deprecated. Please use verbosity instead. "
+ + "0 means printing running messages, 1 means silent mode [default: 1]");
+ opts.addOption("verbosity", true, "Verbosity of printing messages. "
+ + "Choices: 0 (silent), 1 (warning), 2 (info), 3 (debug). [default: 0]");
+ opts.addOption("disable_default_eval_metric", true,
+ "NFlag to disable default metric. Set to >0 to disable. [default: 0]");
+ opts.addOption("num_pbuffer", true,
+ "Size of prediction buffer [default: set automatically by xgboost]");
+ opts.addOption("num_feature", true,
+ "Feature dimension used in boosting [default: set automatically by xgboost]");
+
+ /** Parameters among Boosters */
+ opts.addOption("lambda", "reg_lambda", true,
+ "L2 regularization term on weights. Increasing this value will make model more conservative."
+ + " [default: 1.0 for gbtree, 0.0 for gblinear]");
+ opts.addOption("alpha", "reg_alpha", true,
+ "L1 regularization term on weights. Increasing this value will make model more conservative."
+ + " [default: 0.0]");
+ opts.addOption("updater", true,
+ "A comma-separated string that defines the sequence of tree updaters to run. "
+ + "For a full list of valid inputs, please refer to XGBoost Parameters."
+ + " [default: 'grow_colmaker,prune' for gbtree, 'shotgun' for gblinear]");
+
+ /** Parameters for Tree Booster */
+ opts.addOption("eta", "learning_rate", true,
+ "Step size shrinkage used in update to prevents overfitting [default: 0.3]");
+ opts.addOption("gamma", "min_split_loss", true,
+ "Minimum loss reduction required to make a further partition on a leaf node of the tree."
+ + " [default: 0.0]");
+ opts.addOption("max_depth", true, "Max depth of decision tree [default: 6]");
+ opts.addOption("min_child_weight", true,
+ "Minimum sum of instance weight (hessian) needed in a child [default: 1.0]");
+ opts.addOption("max_delta_step", true,
+ "Maximum delta step we allow each tree's weight estimation to be [default: 0]");
+ opts.addOption("subsample", true,
+ "Subsample ratio of the training instance in range (0.0,1.0] [default: 1.0]");
+ opts.addOption("colsample_bytree", true,
+ "Subsample ratio of columns when constructing each tree [default: 1.0]");
+ opts.addOption("colsample_bylevel", true,
+ "Subsample ratio of columns for each level [default: 1.0]");
+ opts.addOption("colsample_bynode", true,
+ "Subsample ratio of columns for each node [default: 1.0]");
+ // tree_method
+ opts.addOption("tree_method", true,
+ "The tree construction algorithm used in XGBoost. [default: auto, Choices: auto, exact, approx, hist]");
+ opts.addOption("sketch_eps", true,
+ "This roughly translates into O(1 / sketch_eps) number of bins. \n"
+ + "Compared to directly select number of bins, this comes with theoretical guarantee with sketch accuracy.\n"
+ + "Only used for tree_method=approx. Usually user does not have to tune this. [default: 0.03]");
+ opts.addOption("scale_pos_weight", true,
+ "ontrol the balance of positive and negative weights, useful for unbalanced classes. "
+ + "A typical value to consider: sum(negative instances) / sum(positive instances)"
+ + " [default: 1.0]");
+ opts.addOption("refresh_leaf", true,
+ "This is a parameter of the refresh updater plugin. "
+ + "When this flag is 1, tree leafs as well as tree nodes’ stats are updated. "
+ + "When it is 0, only node stats are updated. [default: 1]");
+ opts.addOption("process_type", true,
+ "A type of boosting process to run. [Choices: default, update]");
+ opts.addOption("grow_policy", true,
+ "Controls a way new nodes are added to the tree. Currently supported only if tree_method is set to hist."
+ + " [default: depthwise, Choices: depthwise, lossguide]");
+ opts.addOption("max_leaves", true,
+ "Maximum number of nodes to be added. Only relevant when grow_policy=lossguide is set. [default: 0]");
+ opts.addOption("max_bin", true,
+ "Maximum number of discrete bins to bucket continuous features. Only used if tree_method is set to hist."
+ + " [default: 256]");
+ opts.addOption("num_parallel_tree", true,
+ "Number of parallel trees constructed during each iteration. This option is used to support boosted random forest. "
+ + "Usually no need to tune (default 1 is enough) for gradient boosting trees."
+ + " [default: 1]");
+
+ /** Parameters for Dart Booster (booster=dart) */
+ opts.addOption("sample_type", true,
+ "Type of sampling algorithm. [Choices: uniform (default), weighted]");
+ opts.addOption("normalize_type", true,
+ "Type of normalization algorithm. [Choices: tree (default), forest]");
+ opts.addOption("rate_drop", true, "Dropout rate in range [0.0, 1.0]. [default: 0.0]");
+ opts.addOption("one_drop", true,
+ "When this flag is enabled, at least one tree is always dropped during the dropout. "
+ + "0 or 1. [default: 0]");
+ opts.addOption("skip_drop", true,
+ "Probability of skipping the dropout procedure during a boosting iteration "
+ + "in range [0.0, 1.0]. [default: 0.0]");
+
+ /** Parameters for Linear Booster (booster=gblinear) */
+ opts.addOption("lambda_bias", true, "L2 regularization term on bias [default: 0.0]");
+ opts.addOption("feature_selector", true, "Feature selection and ordering method. "
+ + "[Choices: cyclic (default), shuffle, random, greedy, thrifty]");
+ opts.addOption("top_k", true,
+ "The number of top features to select in greedy and thrifty feature selector. "
+ + "The value of 0 means using all the features. [default: 0]");
+
+ /** Parameters for Tweedie Regression (objective=reg:tweedie) */
+ opts.addOption("tweedie_variance_power", true,
+ "Parameter that controls the variance of the Tweedie distribution in range [1.0, 2.0]."
+ + " [default: 1.5]");
+
+ /** Learning Task Parameters */
+ opts.addOption("objective", true,
+ "Specifies the learning task and the corresponding learning objective. "
+ + "Examples: reg:linear, reg:logistic, multi:softmax. "
+ + "For a full list of valid inputs, refer to XGBoost Parameters. "
+ + "[default: reg:linear]");
+ opts.addOption("base_score", true,
+ "Initial prediction score of all instances, global bias [default: 0.5]");
+ opts.addOption("eval_metric", true,
+ "Evaluation metrics for validation data. A default metric is assigned according to the objective:\n"
+ + "- rmse: for regression\n" + "- error: for classification\n"
+ + "- map: for ranking\n"
+ + "For a list of valid inputs, see XGBoost Parameters.");
+ opts.addOption("seed", true, "Random number seed. [default: 43]");
+ opts.addOption("num_class", true, "Number of classes to classify");
+
+ return opts;
+ }
+
+ @Nonnull
+ @Override
+ protected CommandLine processOptions(ObjectInspector[] argOIs) throws UDFArgumentException {
+ final CommandLine cl;
+ if (argOIs.length >= 3) {
+ String rawArgs = HiveUtils.getConstString(argOIs, 2);
+ cl = parseOptions(rawArgs);
+ } else {
+ cl = parseOptions(""); // use default options
+ }
+
+ String objective = cl.getOptionValue("objective");
+ if (objective == null) {
+ showHelp("Please provide \"-objective XXX\" option in the 3rd argument.\n\n"
+ + "Here is the list of supported objectives: \n"
+ + " - Regression:\n {reg:squarederror, reg:logistic, reg:gamma, reg:tweedie}\n"
+ + " - Binary classification: {binary:logistic, binary:logitraw, binary:hinge}\n"
+ + " - Multiclass classification:\n {multi:softmax, multi:softprob}\n"
+ + " - Ranking:\n {rank:pairwise, rank:ndcg, rank:map}\n"
+ + " - Other:\n {count:poisson, survival:cox}");
+ }
+ if (objective.equals("reg:squarederror")) {
+ // reg:linear is deprecated synonym of reg:squarederror
+ // however, reg:squarederror is not supported in xgboost-predictor yet
+ // https://github.com/dmlc/xgboost/pull/4267
+ objective = "reg:linear";
+ }
+ final String booster = cl.getOptionValue("booster", "gbtree");
+
+ int numRound = Primitives.parseInt(cl.getOptionValue("num_round"), 10);
+ params.put("num_round", numRound);
+ params.put("maximize_evaluation_metrics",
+ Primitives.parseBoolean(cl.getOptionValue("maximize_evaluation_metrics"), false));
+ params.put("num_early_stopping_rounds",
+ Primitives.parseInt(cl.getOptionValue("num_early_stopping_rounds"), 0));
+ double validationRatio =
+ Primitives.parseDouble(cl.getOptionValue("validation_ratio"), 0.2d);
+ if (validationRatio < 0.d || validationRatio >= 1.d) {
+ throw new UDFArgumentException("Invalid validation_ratio=" + validationRatio);
+ }
+ params.put("validation_ratio", validationRatio);
+
+ /** General parameters */
+ params.put("booster", booster);
+ params.put("silent", Primitives.parseInt(cl.getOptionValue("silent"), 1));
+ params.put("verbosity", Primitives.parseInt(cl.getOptionValue("verbosity"), 0));
+ params.put("nthread", Primitives.parseInt(cl.getOptionValue("nthread"), 1));
+ params.put("disable_default_eval_metric",
+ Primitives.parseInt(cl.getOptionValue("disable_default_eval_metric"), 0));
+ if (cl.hasOption("num_pbuffer")) {
+ params.put("num_pbuffer", Integer.valueOf(cl.getOptionValue("num_pbuffer")));
+ }
+ if (cl.hasOption("num_feature")) {
+ params.put("num_feature", Integer.valueOf(cl.getOptionValue("num_feature")));
+ }
+
+ /** Parameters for Tree Booster (booster=gbtree) */
+ if (booster.equals("gbtree")) {
+ params.put("eta", Primitives.parseDouble(cl.getOptionValue("eta"), 0.3d));
+ params.put("gamma", Primitives.parseDouble(cl.getOptionValue("gamma"), 0.d));
+ params.put("max_depth", Primitives.parseInt(cl.getOptionValue("max_depth"), 6));
+ params.put("min_child_weight",
+ Primitives.parseDouble(cl.getOptionValue("min_child_weight"), 1.d));
+ params.put("max_delta_step",
+ Primitives.parseDouble(cl.getOptionValue("max_delta_step"), 0.d));
+ params.put("subsample", Primitives.parseDouble(cl.getOptionValue("subsample"), 1.d));
+ params.put("colsamle_bytree",
+ Primitives.parseDouble(cl.getOptionValue("colsample_bytree"), 1.d));
+ params.put("colsamle_bylevel",
+ Primitives.parseDouble(cl.getOptionValue("colsamle_bylevel"), 1.d));
+ params.put("colsamle_bynode",
+ Primitives.parseDouble(cl.getOptionValue("colsamle_bynode"), 1.d));
+ params.put("lambda", Primitives.parseDouble(cl.getOptionValue("lambda"), 1.d));
+ params.put("alpha", Primitives.parseDouble(cl.getOptionValue("alpha"), 0.d));
+ params.put("tree_method", cl.getOptionValue("tree_method", "auto"));
+ params.put("sketch_eps",
+ Primitives.parseDouble(cl.getOptionValue("sketch_eps"), 0.03d));
+ params.put("scale_pos_weight",
+ Primitives.parseDouble(cl.getOptionValue("scale_pos_weight"), 1.d));
+ params.put("updater", cl.getOptionValue("updater", "grow_colmaker,prune"));
+ params.put("refresh_leaf", Primitives.parseInt(cl.getOptionValue("refresh_leaf"), 1));
+ params.put("process_type", cl.getOptionValue("process_type", "default"));
+ params.put("grow_policy", cl.getOptionValue("grow_policy", "depthwise"));
+ params.put("max_leaves", Primitives.parseInt(cl.getOptionValue("max_leaves"), 0));
+ params.put("max_bin", Primitives.parseInt(cl.getOptionValue("max_bin"), 256));
+ params.put("num_parallel_tree",
+ Primitives.parseInt(cl.getOptionValue("num_parallel_tree"), 1));
+ }
+
+ /** Parameters for Dart Booster (booster=dart) */
+ if (booster.equals("dart")) {
+ params.put("sample_type", cl.getOptionValue("sample_type", "uniform"));
+ params.put("normalize_type", cl.getOptionValue("normalize_type", "tree"));
+ params.put("rate_drop", Primitives.parseDouble(cl.getOptionValue("rate_drop"), 0.d));
+ params.put("one_drop", Primitives.parseInt(cl.getOptionValue("one_drop"), 0));
+ params.put("skip_drop", Primitives.parseDouble(cl.getOptionValue("skip_drop"), 0.d));
+ }
+
+ /** Parameters for Linear Booster (booster=gblinear) */
+ if (booster.equals("gblinear")) {
+ params.put("lambda", Primitives.parseDouble(cl.getOptionValue("lambda"), 0.d));
+ params.put("lambda_bias",
+ Primitives.parseDouble(cl.getOptionValue("lambda_bias"), 0.d));
+ params.put("alpha", Primitives.parseDouble(cl.getOptionValue("alpha"), 0.d));
+ params.put("updater", cl.getOptionValue("updater", "shotgun"));
+ params.put("feature_selector", cl.getOptionValue("feature_selector", "cyclic"));
+ params.put("top_k", Primitives.parseInt(cl.getOptionValue("top_k"), 0));
+ }
+
+ /** Parameters for Tweedie Regression (objective=reg:tweedie) */
+ if (objective.equals("reg:tweedie")) {
+ params.put("tweedie_variance_power",
+ Primitives.parseDouble(cl.getOptionValue("tweedie_variance_power"), 1.5d));
+ }
+
+ /** Parameters for Poisson Regression (objective=count:poisson) */
+ if (objective.equals("count:poisson")) {
+ // max_delta_step is set to 0.7 by default in poisson regression (used to safeguard optimization)
+ params.put("max_delta_step",
+ Primitives.parseDouble(cl.getOptionValue("max_delta_step"), 0.7d));
+ }
+
+ /** Learning Task Parameters */
+ params.put("objective", objective);
+ params.put("base_score", Primitives.parseDouble(cl.getOptionValue("base_score"), 0.5d));
+ if (cl.hasOption("eval_metric")) {
+ params.put("eval_metric", cl.getOptionValue("eval_metric"));
+ }
+ params.put("seed", Primitives.parseLong(cl.getOptionValue("seed"), 43L));
+
+ if (cl.hasOption("num_class")) {
+ this.numClass = Integer.parseInt(cl.getOptionValue("num_class"));
+ params.put("num_class", numClass);
+ } else {
+ if (objective.startsWith("multi:")) {
+ throw new UDFArgumentException(
+ "-num_class is required for multiclass classification");
+ }
+ }
+
+ if (logger.isInfoEnabled()) {
+ logger.info("XGboost training hyperparameters: " + params.toString());
+ }
+
+ this.objectiveType = ObjectiveType.resolve(objective);
+
+ return cl;
+ }
+
+ @Override
+ public StructObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
+ throws UDFArgumentException {
+ if (argOIs.length != 2 && argOIs.length != 3) {
+ showHelp("Invalid argment length=" + argOIs.length);
+ }
+ processOptions(argOIs);
+
+ ListObjectInspector listOI = HiveUtils.asListOI(argOIs, 0);
+ ObjectInspector elemOI = listOI.getListElementObjectInspector();
+ this.featureListOI = listOI;
+ if (HiveUtils.isNumberOI(elemOI)) {
+ this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
+ this.denseInput = true;
+ this.matrixBuilder = new DenseDMatrixBuilder(8192);
+ } else if (HiveUtils.isStringOI(elemOI)) {
+ this.featureElemOI = HiveUtils.asStringOI(elemOI);
+ this.denseInput = false;
+ this.matrixBuilder = new SparseDMatrixBuilder(8192);
+ } else {
+ throw new UDFArgumentException(
+ "train_xgboost takes array<double> or array<string> for the first argument: "
+ + listOI.getTypeName());
+ }
+ this.targetOI = HiveUtils.asDoubleCompatibleOI(argOIs, 1);
+ this.labels = new FloatArrayList(1024);
+
+ final List<String> fieldNames = new ArrayList<>(2);
+ final List<ObjectInspector> fieldOIs = new ArrayList<>(2);
+ fieldNames.add("model_id");
+ fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ fieldNames.add("model");
+ fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
+ return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
+ }
+
+ /** To validate target range, overrides this method */
+ protected float processTargetValue(final float target) throws HiveException {
+ switch (objectiveType) {
+ case binary: {
+ if (target != -1 && target != 0 && target != 1) {
+ throw new UDFArgumentException(
+ "Invalid label value for classification: " + target);
+ }
+ return target > 0.f ? 1.f : 0.f;
+ }
+ case multiclass: {
+ final int clazz = (int) target;
+ if (clazz != target) {
+ throw new UDFArgumentException(
+ "Invalid target value for class label: " + target);
+ }
+ if (clazz < 0 || clazz >= numClass) {
+ throw new UDFArgumentException("target must be {0.0, ..., "
+ + String.format("%.1f", (numClass - 1.0)) + "}: " + target);
+ }
+ return target;
+ }
+ default:
+ return target;
+ }
+ }
+
+ @Override
+ public void process(@Nonnull Object[] args) throws HiveException {
+ if (args[0] == null) {
+ throw new HiveException("array<double> features was null");
+ }
+ parseFeatures(args[0], matrixBuilder);
+
+ float target = PrimitiveObjectInspectorUtils.getFloat(args[1], targetOI);
+ labels.add(processTargetValue(target));
+ }
+
+ private void parseFeatures(@Nonnull final Object argObj,
+ @Nonnull final DMatrixBuilder builder) {
+ if (denseInput) {
+ final int length = featureListOI.getListLength(argObj);
+ for (int i = 0; i < length; i++) {
+ Object o = featureListOI.getListElement(argObj, i);
+ if (o == null) {
+ continue;
+ }
+ float v = PrimitiveObjectInspectorUtils.getFloat(o, featureElemOI);
+ builder.nextColumn(i, v);
+ }
+ } else {
+ final int length = featureListOI.getListLength(argObj);
+ for (int i = 0; i < length; i++) {
+ Object o = featureListOI.getListElement(argObj, i);
+ if (o == null) {
+ continue;
+ }
+ String fv = o.toString();
+ builder.nextColumn(fv);
+ }
+ }
+ builder.nextRow();
+ }
+
+ @Override
+ public void close() throws HiveException {
+ DMatrix dmatrix = null;
+ Booster booster = null;
+ try {
+ dmatrix = matrixBuilder.buildMatrix(labels.toArray(true));
+ this.matrixBuilder = null;
+ this.labels = null;
+
+ final int round = OptionUtils.getInt(params, "num_round");
+ final int earlyStoppingRounds = OptionUtils.getInt(params, "num_early_stopping_rounds");
+ if (earlyStoppingRounds > 0) {
+ double validationRatio = OptionUtils.getDouble(params, "validation_ratio");
+ long seed = OptionUtils.getLong(params, "seed");
+
+ int numRows = (int) dmatrix.rowNum();
+ int[] rows = MathUtils.permutation(numRows);
+ ArrayUtils.shuffle(rows, new Random(seed));
+
+ int numTest = (int) (numRows * validationRatio);
+ DMatrix dtrain = null, dtest = null;
+ try {
+ dtest = dmatrix.slice(Arrays.copyOf(rows, numTest));
+ dtrain = dmatrix.slice(Arrays.copyOfRange(rows, numTest, rows.length));
+ booster = train(dtrain, dtest, round, earlyStoppingRounds, params);
+ } finally {
+ XGBoostUtils.close(dtrain);
+ XGBoostUtils.close(dtest);
+ }
+ } else {
+ booster = train(dmatrix, round, params);
+ }
+ onFinishTraining(booster);
+
+ // Output the built model
+ String modelId = generateUniqueModelId();
+ Text predModel = XGBoostUtils.serializeBooster(booster);
+
+ logger.info("model_id:" + modelId.toString() + ", size:" + predModel.getLength());
+ forward(new Object[] {modelId, predModel});
+ } catch (Throwable e) {
+ throw new HiveException(e);
+ } finally {
+ XGBoostUtils.close(dmatrix);
+ XGBoostUtils.close(booster);
+ }
+ }
+
+ @VisibleForTesting
+ protected void onFinishTraining(@Nonnull Booster booster) {}
+
+ @Nonnull
+ private static Booster train(@Nonnull final DMatrix dtrain, @Nonnegative final int round,
+ @Nonnull final Map<String, Object> params)
+ throws NoSuchMethodException, IllegalAccessException, InvocationTargetException,
+ InstantiationException, XGBoostError {
+ final Booster booster = XGBoostUtils.createBooster(dtrain, params);
+ for (int iter = 0; iter < round; iter++) {
+ booster.update(dtrain, iter);
+ }
+ return booster;
+ }
+
+ @Nonnull
+ private static Booster train(@Nonnull final DMatrix dtrain, @Nonnull final DMatrix dtest,
+ @Nonnegative final int round, @Nonnegative final int earlyStoppingRounds,
+ @Nonnull final Map<String, Object> params)
+ throws NoSuchMethodException, IllegalAccessException, InvocationTargetException,
+ InstantiationException, XGBoostError {
+ final Booster booster = XGBoostUtils.createBooster(dtrain, params);
+
+ final boolean maximizeEvaluationMetrics =
+ OptionUtils.getBoolean(params, "maximize_evaluation_metrics");
+ float bestScore = maximizeEvaluationMetrics ? -Float.MAX_VALUE : Float.MAX_VALUE;
+ int bestIteration = 0;
+
+ final float[] metricsOut = new float[1];
+ for (int iter = 0; iter < round; iter++) {
+ booster.update(dtrain, iter);
+
+ String evalInfo =
+ booster.evalSet(new DMatrix[] {dtest}, new String[] {"test"}, iter, metricsOut);
+ logger.info(evalInfo);
+
+ final float score = metricsOut[0];
+ if (maximizeEvaluationMetrics) {
+ // Update best score if the current score is better (no update when equal)
+ if (score > bestScore) {
+ bestScore = score;
+ bestIteration = iter;
+ }
+ } else {
+ if (score < bestScore) {
+ bestScore = score;
+ bestIteration = iter;
+ }
+ }
+
+ if (shouldEarlyStop(earlyStoppingRounds, iter, bestIteration)) {
+ logger.info(
+ String.format("early stopping after %d rounds away from the best iteration",
+ earlyStoppingRounds));
+ break;
+ }
+ }
+
+ return booster;
+ }
+
+ private static boolean shouldEarlyStop(final int earlyStoppingRounds, final int iter,
+ final int bestIteration) {
+ return iter - bestIteration >= earlyStoppingRounds;
+ }
+
+ @Nonnull
+ private static String generateUniqueModelId() {
+ return "xgbmodel-" + HadoopUtils.getUniqueTaskIdString();
+ }
+
+}
diff --git a/xgboost/src/main/java/hivemall/xgboost/XGBoostUtils.java b/xgboost/src/main/java/hivemall/xgboost/XGBoostUtils.java
deleted file mode 100644
index 20c44bc..0000000
--- a/xgboost/src/main/java/hivemall/xgboost/XGBoostUtils.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package hivemall.xgboost;
-
-import ml.dmlc.xgboost4j.LabeledPoint;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Properties;
-
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-
-public final class XGBoostUtils {
-
- private XGBoostUtils() {}
-
- /** Transform List<String> inputs into a XGBoost input format */
- @Nullable
- public static LabeledPoint parseFeatures(final double target,
- @Nonnull final String[] features) {
- final int size = features.length;
- if (size == 0) {
- return null;
- }
-
- final int[] indices = new int[size];
- final float[] values = new float[size];
- for (int i = 0; i < size; i++) {
- if (features[i] == null) {
- continue;
- }
- final String str = features[i];
- final int pos = str.indexOf(':');
- if (pos >= 1) {
- indices[i] = Integer.parseInt(str.substring(0, pos));
- values[i] = Float.parseFloat(str.substring(pos + 1));
- }
- }
-
-
- return new LabeledPoint((float) target, indices, values);
- }
-
- @Nonnull
- public static String getVersion() throws HiveException {
- Properties props = new Properties();
- try (InputStream versionResourceFile =
- Thread.currentThread().getContextClassLoader().getResourceAsStream(
- "xgboost4j-version.properties")) {
- props.load(versionResourceFile);
- } catch (IOException e) {
- throw new HiveException("Failed to load xgboost4j-version.properties", e);
- }
- return props.getProperty("version", "<unknown>");
- }
-
-}
diff --git a/xgboost/src/main/java/hivemall/xgboost/XGBoostVersionUDF.java b/xgboost/src/main/java/hivemall/xgboost/XGBoostVersionUDF.java
index 0c08295..a3af74b 100644
--- a/xgboost/src/main/java/hivemall/xgboost/XGBoostVersionUDF.java
+++ b/xgboost/src/main/java/hivemall/xgboost/XGBoostVersionUDF.java
@@ -18,6 +18,8 @@
*/
package hivemall.xgboost;
+import hivemall.xgboost.utils.XGBoostUtils;
+
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
diff --git a/xgboost/src/main/java/hivemall/xgboost/classification/XGBoostBinaryClassifierUDTF.java b/xgboost/src/main/java/hivemall/xgboost/classification/XGBoostBinaryClassifierUDTF.java
deleted file mode 100644
index a0409d7..0000000
--- a/xgboost/src/main/java/hivemall/xgboost/classification/XGBoostBinaryClassifierUDTF.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package hivemall.xgboost.classification;
-
-import hivemall.xgboost.XGBoostBaseUDTF;
-
-import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-
-/**
- * A XGBoost binary classification and the document is as follows; -
- * https://github.com/dmlc/xgboost/tree/master/demo/binary_classification
- */
-@Description(name = "train_xgboost_classifier",
- value = "_FUNC_(array<string> features, double target [, string options])"
- + " - Returns a relation consisting of <string model_id, array<byte> pred_model>")
-public final class XGBoostBinaryClassifierUDTF extends XGBoostBaseUDTF {
-
- public XGBoostBinaryClassifierUDTF() {
- super();
- }
-
- {
- // Settings for binary classification
- params.put("objective", "binary:logistic");
- params.put("eval_metric", "error");
- }
-
- @Override
- protected void checkTargetValue(final double target) throws HiveException {
- if (!(Double.compare(target, 0.0) == 0 || Double.compare(target, 1.0) == 0)) {
- throw new HiveException("target must be 0.0 or 1.0: " + target);
- }
- }
-
-}
diff --git a/xgboost/src/main/java/hivemall/xgboost/classification/XGBoostMulticlassClassifierUDTF.java b/xgboost/src/main/java/hivemall/xgboost/classification/XGBoostMulticlassClassifierUDTF.java
deleted file mode 100644
index cb4c45c..0000000
--- a/xgboost/src/main/java/hivemall/xgboost/classification/XGBoostMulticlassClassifierUDTF.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package hivemall.xgboost.classification;
-
-import hivemall.xgboost.XGBoostBaseUDTF;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-
-/**
- * A XGBoost multiclass classification and the document is as follows; -
- * https://github.com/dmlc/xgboost/tree/master/demo/multiclass_classification
- */
-@Description(name = "train_multiclass_xgboost_classifier",
- value = "_FUNC_(string[] features, double target [, string options]) - Returns a relation consisting of <string model_id, array<byte> pred_model>")
-public final class XGBoostMulticlassClassifierUDTF extends XGBoostBaseUDTF {
-
- public XGBoostMulticlassClassifierUDTF() {
- super();
- }
-
- {
- // Settings for multiclass classification
- params.put("objective", "multi:softprob");
- params.put("eval_metric", "error");
- params.put("num_class", 2);
- }
-
- @Override
- protected Options getOptions() {
- Options opts = super.getOptions();
- opts.addOption("num_class", true, "Number of classes to classify");
- return opts;
- }
-
- @Override
- protected CommandLine processOptions(ObjectInspector[] argOIs) throws UDFArgumentException {
- final CommandLine cli = super.processOptions(argOIs);
- if (cli != null) {
- if (cli.hasOption("num_class")) {
- int _num_class = Integer.valueOf(cli.getOptionValue("num_class"));
- if (_num_class < 2) {
- throw new UDFArgumentException(
- "num_class must be greater than 1: " + _num_class);
- }
- params.put("num_class", _num_class);
- }
- }
- return cli;
- }
-
- @Override
- protected void checkTargetValue(final double target) throws HiveException {
- double num_class = ((Integer) params.get("num_class")).doubleValue();
- if (target < 0.0 || target > num_class
- || Double.compare(target - Math.floor(target), 0.0) != 0) {
- throw new HiveException("target must be {0.0, ..., "
- + String.format("%.1f", (num_class - 1.0)) + "}: " + target);
- }
- }
-
-}
diff --git a/xgboost/src/main/java/hivemall/xgboost/regression/XGBoostRegressionUDTF.java b/xgboost/src/main/java/hivemall/xgboost/regression/XGBoostRegressionUDTF.java
deleted file mode 100644
index c018c33..0000000
--- a/xgboost/src/main/java/hivemall/xgboost/regression/XGBoostRegressionUDTF.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package hivemall.xgboost.regression;
-
-import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-
-import hivemall.xgboost.XGBoostBaseUDTF;
-
-/**
- * A XGBoost regression and the document is as follows; -
- * https://github.com/dmlc/xgboost/tree/master/demo/regression
- */
-@Description(name = "train_xgboost_regr",
- value = "_FUNC_(string[] features, double target [, string options]) - Returns a relation consisting of <string model_id, array<byte> pred_model>")
-public final class XGBoostRegressionUDTF extends XGBoostBaseUDTF {
-
- public XGBoostRegressionUDTF() {
- super();
- }
-
- {
- // Settings for logistic regression
- params.put("objective", "reg:logistic");
- params.put("eval_metric", "rmse");
- }
-
- @Override
- protected void checkTargetValue(final double target) throws HiveException {
- if (target < 0.0 || target > 1.0) {
- throw new HiveException("target must be in range 0 to 1: " + target);
- }
- }
-
-}
diff --git a/xgboost/src/main/java/hivemall/xgboost/tools/XGBoostMulticlassPredictUDTF.java b/xgboost/src/main/java/hivemall/xgboost/tools/XGBoostMulticlassPredictUDTF.java
deleted file mode 100644
index b80f95a..0000000
--- a/xgboost/src/main/java/hivemall/xgboost/tools/XGBoostMulticlassPredictUDTF.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package hivemall.xgboost.tools;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.annotation.Nonnull;
-
-import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-
-import hivemall.utils.lang.Preconditions;
-
-@Description(name = "xgboost_multiclass_predict",
- value = "_FUNC_(string rowid, string[] features, string model_id, array<byte> pred_model [, string options]) "
- + "- Returns a prediction result as (string rowid, string label, float probability)")
-public final class XGBoostMulticlassPredictUDTF extends hivemall.xgboost.XGBoostPredictUDTF {
-
- public XGBoostMulticlassPredictUDTF() {
- super();
- }
-
- /** Return (string rowid, int label, float probability) as a result */
- @Override
- protected StructObjectInspector getReturnOI() {
- final List<String> fieldNames = new ArrayList<>(3);
- final List<ObjectInspector> fieldOIs = new ArrayList<>(3);
- fieldNames.add("rowid");
- fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
- fieldNames.add("label");
- fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
- fieldNames.add("probability");
- fieldOIs.add(PrimitiveObjectInspectorFactory.javaFloatObjectInspector);
-
- return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
- }
-
- @Override
- protected void forwardPredicted(@Nonnull final List<LabeledPointWithRowId> testData,
- @Nonnull final float[][] predicted) throws HiveException {
- Preconditions.checkArgument(predicted.length == testData.size(), HiveException.class);
-
- final Object[] forwardObj = new Object[3];
- for (int i = 0, size = testData.size(); i < size; i++) {
- final float[] predicted_i = predicted[i];
- String rowId = testData.get(i).getRowId();
- forwardObj[0] = rowId;
-
- assert (predicted_i.length > 1);
- for (int j = 0; j < predicted_i.length; j++) {
- forwardObj[1] = String.valueOf(j);
- float prob = predicted_i[j];
- forwardObj[2] = Float.valueOf(prob);
- forward(forwardObj);
- }
- }
- }
-
-}
diff --git a/xgboost/src/main/java/hivemall/xgboost/tools/XGBoostPredictUDTF.java b/xgboost/src/main/java/hivemall/xgboost/tools/XGBoostPredictUDTF.java
deleted file mode 100644
index 85d0fb1..0000000
--- a/xgboost/src/main/java/hivemall/xgboost/tools/XGBoostPredictUDTF.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package hivemall.xgboost.tools;
-
-import hivemall.utils.lang.Preconditions;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.annotation.Nonnull;
-
-import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-
-@Description(name = "xgboost_predict",
- value = "_FUNC_(string rowid, string[] features, string model_id, array<byte> pred_model [, string options]) "
- + "- Returns a prediction result as (string rowid, float predicted)")
-public final class XGBoostPredictUDTF extends hivemall.xgboost.XGBoostPredictUDTF {
-
- public XGBoostPredictUDTF() {
- super();
- }
-
- /** Return (string rowid, float predicted) as a result */
- @Override
- protected StructObjectInspector getReturnOI() {
- final List<String> fieldNames = new ArrayList<>(2);
- final List<ObjectInspector> fieldOIs = new ArrayList<>(2);
- fieldNames.add("rowid");
- fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
- fieldNames.add("predicted");
- fieldOIs.add(PrimitiveObjectInspectorFactory.javaFloatObjectInspector);
-
- return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
- }
-
- @Override
- protected void forwardPredicted(@Nonnull final List<LabeledPointWithRowId> testData,
- @Nonnull final float[][] predicted) throws HiveException {
- Preconditions.checkArgument(predicted.length == testData.size(), HiveException.class);
-
- final Object[] forwardObj = new Object[2];
- for (int i = 0, size = testData.size(); i < size; i++) {
- assert (predicted[i].length == 1);
-
- final String rowId = testData.get(i).getRowId();
- float p = predicted[i][0];
- forwardObj[0] = rowId;
- forwardObj[1] = p;
-
- forward(forwardObj);
- }
- }
-
-}
diff --git a/xgboost/src/main/java/hivemall/xgboost/utils/DMatrixBuilder.java b/xgboost/src/main/java/hivemall/xgboost/utils/DMatrixBuilder.java
new file mode 100644
index 0000000..bf11020
--- /dev/null
+++ b/xgboost/src/main/java/hivemall/xgboost/utils/DMatrixBuilder.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.xgboost.utils;
+
+import ml.dmlc.xgboost4j.java.DMatrix;
+import ml.dmlc.xgboost4j.java.XGBoostError;
+
+import javax.annotation.Nonnegative;
+import javax.annotation.Nonnull;
+
+public abstract class DMatrixBuilder {
+
+ public DMatrixBuilder() {}
+
+ protected static final void checkColIndex(final int col) {
+ if (col < 0) {
+ throw new IllegalArgumentException("Found negative column index: " + col);
+ }
+ }
+
+ public void nextRow(@Nonnull final float[] row) {
+ for (int col = 0; col < row.length; col++) {
+ nextColumn(col, row[col]);
+ }
+ nextRow();
+ }
+
+ public void nextRow(@Nonnull final String[] row) {
+ for (String col : row) {
+ if (col == null) {
+ continue;
+ }
+ nextColumn(col);
+ }
+ nextRow();
+ }
+
+ public void nextRow(@Nonnull final String[] row, final int start, final int endEx) {
+ for (int i = start, last = Math.min(endEx, row.length); i < last; i++) {
+ String col = row[i];
+ if (col == null) {
+ continue;
+ }
+ nextColumn(col);
+ }
+ nextRow();
+ }
+
+ @Nonnull
+ public abstract DMatrixBuilder nextRow();
+
+ @Nonnull
+ public abstract DMatrixBuilder nextColumn(@Nonnegative int col, float value);
+
+ /**
+ * @throws IllegalArgumentException
+ * @throws NumberFormatException
+ */
+ @Nonnull
+ public DMatrixBuilder nextColumn(@Nonnull final String col) {
+ final int pos = col.indexOf(':');
+ if (pos == 0) {
+ throw new IllegalArgumentException("Invalid feature value representation: " + col);
+ }
+
+ final String feature;
+ final float value;
+ if (pos > 0) {
+ feature = col.substring(0, pos);
+ String s2 = col.substring(pos + 1);
+ value = Float.parseFloat(s2);
+ } else {
+ feature = col;
+ value = 1.f;
+ }
+
+ if (feature.indexOf(':') != -1) {
+ throw new IllegalArgumentException("Invalid feature format `<index>:<value>`: " + col);
+ }
+
+ int colIndex = Integer.parseInt(feature);
+ if (colIndex < 0) {
+ throw new IllegalArgumentException(
+ "Col index MUST be greater than or equals to 0: " + colIndex);
+ }
+
+ return nextColumn(colIndex, value);
+ }
+
+ @Nonnull
+ public abstract DMatrix buildMatrix(@Nonnull float[] labels) throws XGBoostError;
+
+}
diff --git a/xgboost/src/main/java/hivemall/xgboost/utils/DenseDMatrixBuilder.java b/xgboost/src/main/java/hivemall/xgboost/utils/DenseDMatrixBuilder.java
new file mode 100644
index 0000000..7129059
--- /dev/null
+++ b/xgboost/src/main/java/hivemall/xgboost/utils/DenseDMatrixBuilder.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.xgboost.utils;
+
+import hivemall.utils.collections.arrays.SparseFloatArray;
+import ml.dmlc.xgboost4j.java.DMatrix;
+import ml.dmlc.xgboost4j.java.XGBoostError;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import javax.annotation.Nonnegative;
+import javax.annotation.Nonnull;
+
+public final class DenseDMatrixBuilder extends DMatrixBuilder {
+
+ @Nonnull
+ private final List<float[]> rows;
+ private int maxNumColumns;
+
+ @Nonnull
+ private final SparseFloatArray rowProbe;
+
+ public DenseDMatrixBuilder(@Nonnegative int initSize) {
+ super();
+ this.rows = new ArrayList<float[]>(initSize);
+ this.maxNumColumns = 0;
+ this.rowProbe = new SparseFloatArray(32);
+ }
+
+ @Override
+ public DenseDMatrixBuilder nextColumn(@Nonnegative final int col, final float value) {
+ checkColIndex(col);
+
+ this.maxNumColumns = Math.max(col + 1, maxNumColumns);
+ if (value == 0.d) {
+ return this;
+ }
+ rowProbe.put(col, value);
+ return this;
+ }
+
+ @Override
+ public DenseDMatrixBuilder nextRow() {
+ float[] row = rowProbe.toArray();
+ rowProbe.clear();
+ rows.add(row);
+ return this;
+ }
+
+ @Override
+ public DMatrix buildMatrix(@Nonnull float[] labels) throws XGBoostError {
+ final int numRows = rows.size();
+ if (labels.length != numRows) {
+ throw new XGBoostError(
+ String.format("labels.length does not match to nrows. labels.length=%d, nrows=%d",
+ labels.length, numRows));
+ }
+
+ final float[] data = new float[numRows * maxNumColumns];
+ Arrays.fill(data, Float.NaN);
+ for (int i = 0; i < numRows; i++) {
+ final float[] row = rows.get(i);
+ final int rowPtr = i * maxNumColumns;
+ for (int j = 0; j < row.length; j++) {
+ int ij = rowPtr + j;
+ data[ij] = row[j];
+ }
+ }
+
+ DMatrix matrix = new DMatrix(data, numRows, maxNumColumns, Float.NaN);
+ matrix.setLabel(labels);
+ return matrix;
+ }
+
+}
diff --git a/xgboost/src/main/java/hivemall/xgboost/NativeLibLoader.java b/xgboost/src/main/java/hivemall/xgboost/utils/NativeLibLoader.java
similarity index 99%
rename from xgboost/src/main/java/hivemall/xgboost/NativeLibLoader.java
rename to xgboost/src/main/java/hivemall/xgboost/utils/NativeLibLoader.java
index bf3ea1f..ffe33aa 100644
--- a/xgboost/src/main/java/hivemall/xgboost/NativeLibLoader.java
+++ b/xgboost/src/main/java/hivemall/xgboost/utils/NativeLibLoader.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package hivemall.xgboost;
+package hivemall.xgboost.utils;
import java.io.File;
import java.io.FileInputStream;
diff --git a/xgboost/src/main/java/hivemall/xgboost/utils/SparseDMatrixBuilder.java b/xgboost/src/main/java/hivemall/xgboost/utils/SparseDMatrixBuilder.java
index 6f8e739..c5914bf 100644
--- a/xgboost/src/main/java/hivemall/xgboost/utils/SparseDMatrixBuilder.java
+++ b/xgboost/src/main/java/hivemall/xgboost/utils/SparseDMatrixBuilder.java
@@ -24,10 +24,16 @@ import matrix4j.utils.collections.lists.IntArrayList;
import ml.dmlc.xgboost4j.java.DMatrix;
import ml.dmlc.xgboost4j.java.XGBoostError;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
import javax.annotation.Nonnegative;
import javax.annotation.Nonnull;
-public class SparseDMatrixBuilder {
+public final class SparseDMatrixBuilder extends DMatrixBuilder {
+
+ private final boolean sortRequired;
@Nonnull
private final LongArrayList rowPointers;
@@ -36,28 +42,43 @@ public class SparseDMatrixBuilder {
@Nonnull
private final FloatArrayList values;
+ @Nonnull
+ private final List<ColValue> colCache;
+
private int maxNumColumns;
public SparseDMatrixBuilder(@Nonnegative int initSize) {
+ this(initSize, true);
+ }
+
+ public SparseDMatrixBuilder(@Nonnegative int initSize, boolean sortRequired) {
+ super();
+ this.sortRequired = sortRequired;
this.rowPointers = new LongArrayList(initSize + 1);
rowPointers.add(0);
this.columnIndices = new IntArrayList(initSize);
this.values = new FloatArrayList(initSize);
+ this.colCache = new ArrayList<>(32);
this.maxNumColumns = 0;
}
+ @Nonnull
public SparseDMatrixBuilder nextRow() {
+ if (sortRequired) {
+ Collections.sort(colCache);
+ }
+ for (ColValue e : colCache) {
+ columnIndices.add(e.col);
+ values.add(e.value);
+ }
+ colCache.clear();
+
int ptr = values.size();
rowPointers.add(ptr);
return this;
}
- private static final void checkColIndex(final int col) {
- if (col < 0) {
- throw new IllegalArgumentException("Found negative column index: " + col);
- }
- }
-
+ @Nonnull
public SparseDMatrixBuilder nextColumn(@Nonnegative int col, float value) {
checkColIndex(col);
@@ -66,14 +87,37 @@ public class SparseDMatrixBuilder {
return this;
}
- columnIndices.add(col);
- values.add(value);
+ colCache.add(new ColValue(col, value));
return this;
}
@Nonnull
- public DMatrix buildMatrix() throws XGBoostError {
- return new DMatrix(rowPointers.toArray(true), columnIndices.toArray(true),
+ public DMatrix buildMatrix(@Nonnull float[] labels) throws XGBoostError {
+ DMatrix matrix = new DMatrix(rowPointers.toArray(true), columnIndices.toArray(true),
values.toArray(true), DMatrix.SparseType.CSR, maxNumColumns);
+ matrix.setLabel(labels);
+ return matrix;
+ }
+
+ private static final class ColValue implements Comparable<ColValue> {
+ final int col;
+ final float value;
+
+ ColValue(int col, float value) {
+ this.col = col;
+ this.value = value;
+ }
+
+ @Override
+ public int compareTo(ColValue o) {
+ return Integer.compare(col, o.col);
+ }
+
+ @Override
+ public String toString() {
+ return "[column=" + col + ", value=" + value + ']';
+ }
+
}
+
}
diff --git a/xgboost/src/main/java/hivemall/xgboost/utils/XGBoostUtils.java b/xgboost/src/main/java/hivemall/xgboost/utils/XGBoostUtils.java
new file mode 100644
index 0000000..75d0fcc
--- /dev/null
+++ b/xgboost/src/main/java/hivemall/xgboost/utils/XGBoostUtils.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.xgboost.utils;
+
+import biz.k11i.xgboost.Predictor;
+import biz.k11i.xgboost.util.FVec;
+import hivemall.utils.io.FastByteArrayInputStream;
+import hivemall.utils.io.IOUtils;
+import hivemall.xgboost.XGBoostBatchPredictUDTF.LabeledPointWithRowId;
+import ml.dmlc.xgboost4j.LabeledPoint;
+import ml.dmlc.xgboost4j.java.Booster;
+import ml.dmlc.xgboost4j.java.DMatrix;
+import ml.dmlc.xgboost4j.java.XGBoost;
+import ml.dmlc.xgboost4j.java.XGBoostError;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.Text;
+
+public final class XGBoostUtils {
+
+ private XGBoostUtils() {}
+
+ @Nonnull
+ public static String getVersion() throws HiveException {
+ Properties props = new Properties();
+ try (InputStream versionResourceFile =
+ Thread.currentThread()
+ .getContextClassLoader()
+ .getResourceAsStream("xgboost4j-version.properties")) {
+ props.load(versionResourceFile);
+ } catch (IOException e) {
+ throw new HiveException("Failed to load xgboost4j-version.properties", e);
+ }
+ return props.getProperty("version", "<unknown>");
+ }
+
+ @Nonnull
+ public static DMatrix createDMatrix(@Nonnull final List<LabeledPointWithRowId> data)
+ throws XGBoostError {
+ final List<LabeledPoint> points = new ArrayList<>(data.size());
+ for (LabeledPointWithRowId d : data) {
+ points.add(d);
+ }
+ return new DMatrix(points.iterator(), "");
+ }
+
+ @Nonnull
+ public static Booster createBooster(@Nonnull DMatrix matrix,
+ @Nonnull Map<String, Object> params) throws NoSuchMethodException, XGBoostError,
+ IllegalAccessException, InvocationTargetException, InstantiationException {
+ Class<?>[] args = {Map.class, DMatrix[].class};
+ Constructor<Booster> ctor = Booster.class.getDeclaredConstructor(args);
+ ctor.setAccessible(true);
+ return ctor.newInstance(new Object[] {params, new DMatrix[] {matrix}});
+ }
+
+ public static void close(@Nullable final DMatrix matrix) {
+ if (matrix == null) {
+ return;
+ }
+ try {
+ matrix.dispose();
+ } catch (Throwable e) {
+ ;
+ }
+ }
+
+ public static void close(@Nullable final Booster booster) {
+ if (booster == null) {
+ return;
+ }
+ try {
+ booster.dispose();
+ } catch (Throwable e) {
+ ;
+ }
+ }
+
+ @Nonnull
+ public static Text serializeBooster(@Nonnull final Booster booster) throws HiveException {
+ try {
+ byte[] b = IOUtils.toCompressedText(booster.toByteArray());
+ return new Text(b);
+ } catch (Throwable e) {
+ throw new HiveException("Failed to serialize a booster", e);
+ }
+ }
+
+ @Nonnull
+ public static Booster deserializeBooster(@Nonnull final Text model) throws HiveException {
+ try {
+ byte[] b = IOUtils.fromCompressedText(model.getBytes(), model.getLength());
+ return XGBoost.loadModel(new FastByteArrayInputStream(b));
+ } catch (Throwable e) {
+ throw new HiveException("Failed to deserialize a booster", e);
+ }
+ }
+
+ @Nonnull
+ public static Predictor loadPredictor(@Nonnull final Text model) throws HiveException {
+ try {
+ byte[] b = IOUtils.fromCompressedText(model.getBytes(), model.getLength());
+ return new Predictor(new FastByteArrayInputStream(b));
+ } catch (Throwable e) {
+ throw new HiveException("Failed to create a predictor", e);
+ }
+ }
+
+ @Nonnull
+ public static FVec parseRowAsFVec(@Nonnull final String[] row, final int start, final int end) {
+ final Map<Integer, Float> map = new HashMap<>((int) (row.length * 1.5));
+ for (int i = start; i < end; i++) {
+ String f = row[i];
+ if (f == null) {
+ continue;
+ }
+ String str = f.toString();
+ final int pos = str.indexOf(':');
+ if (pos < 1) {
+ throw new IllegalArgumentException("Invalid feature format: " + str);
+ }
+ final int index;
+ final float value;
+ try {
+ index = Integer.parseInt(str.substring(0, pos));
+ value = Float.parseFloat(str.substring(pos + 1));
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Failed to parse a feature value: " + str);
+ }
+ map.put(index, value);
+ }
+
+ return FVec.Transformer.fromMap(map);
+ }
+
+}
diff --git a/xgboost/src/test/java/hivemall/TestBase.java b/xgboost/src/test/java/hivemall/TestBase.java
new file mode 100644
index 0000000..11e8582
--- /dev/null
+++ b/xgboost/src/test/java/hivemall/TestBase.java
@@ -0,0 +1,450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall;
+
+import biz.k11i.xgboost.util.FVec;
+import hivemall.utils.collections.lists.FloatArrayList;
+import hivemall.utils.io.IOUtils;
+import hivemall.utils.lang.ArrayUtils;
+import hivemall.utils.math.MathUtils;
+import hivemall.xgboost.utils.DMatrixBuilder;
+import hivemall.xgboost.utils.DenseDMatrixBuilder;
+import hivemall.xgboost.utils.SparseDMatrixBuilder;
+import hivemall.xgboost.utils.XGBoostUtils;
+import ml.dmlc.xgboost4j.java.DMatrix;
+
+import java.io.BufferedReader;
+import java.lang.reflect.Array;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.Random;
+
+import javax.annotation.Nonnull;
+
+import org.junit.Assert;
+
+public abstract class TestBase {
+
+ public static class TestParameter {
+ @Nonnull
+ public final Dataset trainDataset, testDataset;
+ @Nonnull
+ public final EvalMetric evalMetric;
+ @Nonnull
+ public final String hyperParams;
+
+ public TestParameter(Dataset trainDataset, Dataset testDataset, EvalMetric evalMetric,
+ String hyperParams) {
+ this.trainDataset = Objects.requireNonNull(trainDataset);
+ this.testDataset = Objects.requireNonNull(testDataset);
+ this.evalMetric = Objects.requireNonNull(evalMetric);
+ this.hyperParams = Objects.requireNonNull(hyperParams);
+ }
+
+ public static class Builder {
+ private Dataset trainDataset, testDatase;
+ private EvalMetric evalMetric;
+ private String[] hyperParams;
+
+ public Builder trainDataset(Dataset trainDataset) {
+ this.trainDataset = trainDataset;
+ return this;
+ }
+
+ public Builder testDatase(Dataset testDatase) {
+ this.testDatase = testDatase;
+ return this;
+ }
+
+ public Builder metric(EvalMetric metric) {
+ this.evalMetric = metric;
+ return this;
+ }
+
+ public Builder hyperParams(String[] hyperParams) {
+ this.hyperParams = hyperParams;
+ return this;
+ }
+
+ public List<TestParameter> build() {
+ List<TestParameter> result = new ArrayList<>();
+ for (String p : hyperParams) {
+ result.add(new TestParameter(trainDataset, testDatase, evalMetric, p));
+ }
+ return result;
+ }
+ }
+
+ @Nonnull
+ public static List<TestParameter> merge(@Nonnull Builder... builders) {
+ List<TestParameter> result = new ArrayList<>();
+ for (Builder builder : builders) {
+ result.addAll(builder.build());
+ }
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "TestParameter [trainDataset=" + trainDataset + ", testDataset=" + testDataset
+ + ", evalMetric=" + evalMetric + ", hyperParams=" + hyperParams + "]";
+ }
+ }
+
+ public interface EvalMetric {
+
+ void next(double[] predicted, float expected);
+
+ float result();
+
+ void assertExpected();
+ }
+
+ public static class ClassificationError implements EvalMetric {
+
+ final float acceptedErrorRate;
+ int total = 0, errors = 0;
+
+ public ClassificationError(float acceptedErrorRate) {
+ this.acceptedErrorRate = acceptedErrorRate;
+ }
+
+ @Override
+ public void next(double[] predicted, float expected) {
+ Assert.assertEquals(1, predicted.length);
+ int expectedLabel = (expected > 0) ? 1 : 0;
+ double prob = predicted[0];
+ Assert.assertTrue(prob >= 0);
+ Assert.assertTrue(prob <= 1.0);
+ int actuallabel = (prob > 0.5f) ? 1 : 0;
+ if (expectedLabel != actuallabel) {
+ errors++;
+ }
+ total++;
+ }
+
+ @Override
+ public float result() {
+ return errors / (float) total;
+ }
+
+ @Override
+ public void assertExpected() {
+ float errorRate = result();
+ Assert.assertTrue(String.format(
+ "classification error rate expected to be less than or equals to %f but %f",
+ acceptedErrorRate, errorRate), errorRate <= acceptedErrorRate);
+ }
+
+ }
+
+ public static class MultiClassClassificationError implements EvalMetric {
+ final float acceptedErrorRate;
+ int total = 0, errors = 0;
+
+ public MultiClassClassificationError(float acceptedErrorRate) {
+ this.acceptedErrorRate = acceptedErrorRate;
+ }
+
+ @Override
+ public void next(double[] predicted, float expected) {
+ final int actualLabel;
+ if (predicted.length == 1) {//-objective multi:softmax
+ actualLabel = (int) predicted[0];
+ } else {// -objective multi:softprob
+ actualLabel = ArrayUtils.argmax(predicted);
+ }
+
+ int expectedLabel = (int) expected;
+ if (expectedLabel != actualLabel) {
+ errors++;
+ }
+ total++;
+ }
+
+ @Override
+ public float result() {
+ return errors / (float) total;
+ }
+
+ @Override
+ public void assertExpected() {
+ float errorRate = result();
+ Assert.assertTrue(String.format(
+ "classification error rate expected to be less than or equals to %f but %f",
+ acceptedErrorRate, errorRate), errorRate <= acceptedErrorRate);
+ }
+
+ }
+
+ public static class MAE implements EvalMetric {
+ final float ensureLessThan;
+
+ private double diffSum = 0.d;
+ private int count = 0;
+
+ public MAE(float ensureLessThan) {
+ this.ensureLessThan = ensureLessThan;
+ }
+
+ @Override
+ public void next(double[] predicted, float expected) {
+ Assert.assertEquals(1, predicted.length);
+
+ this.diffSum += Math.abs(predicted[0] - expected);
+ this.count += 1;
+ }
+
+ @Override
+ public float result() {
+ return (float) (diffSum / count);
+ }
+
+ @Override
+ public void assertExpected() {
+ float mae = result();
+ Assert.assertTrue(
+ String.format("MAE expected to be less than %f but %f", ensureLessThan, mae),
+ mae < ensureLessThan);
+ }
+
+
+ }
+
+ public static abstract class Dataset {
+
+ @Nonnull
+ private final String datasetUrl;
+ @Nonnull
+ private final String separator;
+
+ public Dataset(@Nonnull String datasetUrl, @Nonnull String separator) {
+ this.datasetUrl = datasetUrl;
+ this.separator = separator;
+ }
+
+ @Nonnull
+ public String getDatasetUrl() {
+ return datasetUrl;
+ }
+
+ public abstract boolean isSparseDataset();
+
+ public void parse(@Nonnull RowProcessor proc) throws Exception {
+ URL url = new URL(datasetUrl);
+ try (BufferedReader reader = IOUtils.bufferedReader(url.openStream())) {
+ String line = reader.readLine();
+ while (line != null) {
+ String[] splitted = line.split(separator);
+ proc.handleRow(splitted);
+ line = reader.readLine();
+ }
+ }
+ }
+
+ public abstract List<Object[]> loadDatasetAsListOfObjects() throws Exception;
+
+ public abstract DMatrix loadDatasetAsDMatrix() throws Exception;
+
+ public abstract List<FVec> loadDatasetAsListOfFVec() throws Exception;
+
+ }
+
+ public interface RowProcessor {
+ void handleRow(String[] splitted) throws Exception;
+ }
+
+ public static class LibsvmDataset extends Dataset {
+
+ public LibsvmDataset(@Nonnull String datasetUrl) {
+ super(datasetUrl, " ");
+ }
+
+ @Override
+ public boolean isSparseDataset() {
+ return true;
+ }
+
+ @Override
+ public List<Object[]> loadDatasetAsListOfObjects() throws Exception {
+ final List<Object[]> dataset = new ArrayList<>();
+
+ RowProcessor proc = new RowProcessor() {
+ @Override
+ public void handleRow(String[] splitted) throws Exception {
+ Object[] row = new Object[2];
+ row[0] = Arrays.asList(Arrays.copyOfRange(splitted, 1, splitted.length));
+ row[1] = Double.parseDouble(splitted[0]);
+ dataset.add(row);
+ }
+
+ };
+ parse(proc);
+
+ return dataset;
+ }
+
+ @Override
+ public DMatrix loadDatasetAsDMatrix() throws Exception {
+ final DMatrixBuilder builder = new SparseDMatrixBuilder(1024, false);
+ final FloatArrayList labels = new FloatArrayList(1024);
+
+ RowProcessor proc = new RowProcessor() {
+ @Override
+ public void handleRow(String[] splitted) throws Exception {
+ float label = Float.parseFloat(splitted[0]);
+ labels.add(label);
+ builder.nextRow(splitted, 1, splitted.length);
+ }
+ };
+ parse(proc);
+
+ return builder.buildMatrix(labels.toArray());
+ }
+
+ @Override
+ public List<FVec> loadDatasetAsListOfFVec() throws Exception {
+ final List<FVec> dataset = new ArrayList<>();
+
+ RowProcessor proc = new RowProcessor() {
+ @Override
+ public void handleRow(String[] splitted) throws Exception {
+ FVec fv = XGBoostUtils.parseRowAsFVec(splitted, 1, splitted.length);
+ dataset.add(fv);
+ }
+
+ };
+ parse(proc);
+
+ return dataset;
+ }
+ }
+
+ public static class DermatologyDataset extends Dataset {
+ // wc -l dermatology.data > 366
+ private static final int NUM_ROWS = 366;
+
+ @Nonnull
+ private final int[] sliceIndex;
+
+ public DermatologyDataset(boolean train, double trainFrac) {
+ super("https://archive.ics.uci.edu/ml/machine-learning-databases/dermatology/dermatology.data", ",");
+ int[] index = ArrayUtils.shuffle(MathUtils.permutation(NUM_ROWS), new Random(43L));
+ int numTrain = (int) (NUM_ROWS * trainFrac);
+ if (train) {
+ this.sliceIndex = Arrays.copyOfRange(index, 0, numTrain);
+ } else {
+ this.sliceIndex = Arrays.copyOfRange(index, numTrain, index.length);
+ }
+ }
+
+ @Override
+ public boolean isSparseDataset() {
+ return false;
+ }
+
+ @Override
+ public List<Object[]> loadDatasetAsListOfObjects() throws Exception {
+ final List<Object[]> dataset = new ArrayList<>();
+
+ RowProcessor proc = new RowProcessor() {
+ @Override
+ public void handleRow(String[] splitted) throws Exception {
+ final Float[] features = new Float[34];
+ for (int i = 0; i <= 32; i++) {
+ features[i] = Float.parseFloat(splitted[i]);
+ }
+ features[33] = splitted[33].equals("?") ? 0.f : Float.parseFloat(splitted[33]);
+ int label = Integer.parseInt(splitted[34]) - 1;
+ Object[] row = new Object[2];
+ row[0] = Arrays.asList(features);
+ row[1] = Double.valueOf(label);
+ dataset.add(row);
+ }
+
+ };
+ parse(proc);
+
+ return slice(dataset, sliceIndex, Object[].class);
+ }
+
+ @Override
+ public DMatrix loadDatasetAsDMatrix() throws Exception {
+ final DMatrixBuilder builder = new DenseDMatrixBuilder(1024);
+ final FloatArrayList labels = new FloatArrayList(1024);
+
+ RowProcessor proc = new RowProcessor() {
+ @Override
+ public void handleRow(String[] splitted) throws Exception {
+ final float[] features = new float[34];
+ for (int i = 0; i <= 32; i++) {
+ features[i] = Float.parseFloat(splitted[i]);
+ }
+ features[33] = splitted[33].equals("?") ? 0.f : Float.parseFloat(splitted[33]);
+ int label = Integer.parseInt(splitted[34]) - 1;
+
+ labels.add(label);
+ builder.nextRow(features);
+ }
+ };
+ parse(proc);
+
+ return builder.buildMatrix(labels.toArray()).slice(sliceIndex);
+ }
+
+ @Override
+ public List<FVec> loadDatasetAsListOfFVec() throws Exception {
+ final List<FVec> dataset = new ArrayList<>();
+
+ RowProcessor proc = new RowProcessor() {
+ @Override
+ public void handleRow(String[] splitted) throws Exception {
+ final float[] features = new float[34];
+ for (int i = 0; i <= 32; i++) {
+ features[i] = Float.parseFloat(splitted[i]);
+ }
+ features[33] = splitted[33].equals("?") ? 0.f : Float.parseFloat(splitted[33]);
+
+ FVec fv = FVec.Transformer.fromArray(features, false);
+ dataset.add(fv);
+ }
+
+ };
+ parse(proc);
+
+ return slice(dataset, sliceIndex, FVec.class);
+ }
+
+ }
+
+ @SuppressWarnings("unchecked")
+ @Nonnull
+ private static <T> List<T> slice(@Nonnull List<T> original, @Nonnull int[] sliceIndex,
+ @Nonnull Class<?> elemType) {
+ final T[] sliced = (T[]) Array.newInstance(elemType, sliceIndex.length);
+ for (int i = 0; i < sliceIndex.length; i++) {
+ sliced[i] = original.get(sliceIndex[i]);
+ }
+ return Arrays.asList(sliced);
+ }
+
+}
diff --git a/xgboost/src/test/java/hivemall/TestUtils.java b/xgboost/src/test/java/hivemall/TestUtils.java
new file mode 100644
index 0000000..fd8d189
--- /dev/null
+++ b/xgboost/src/test/java/hivemall/TestUtils.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import javax.annotation.Nonnull;
+
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.Collector;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hive.com.esotericsoftware.kryo.Kryo;
+import org.apache.hive.com.esotericsoftware.kryo.io.Input;
+import org.apache.hive.com.esotericsoftware.kryo.io.Output;
+
+public final class TestUtils {
+
+ public static <T extends GenericUDF> void testGenericUDFSerialization(@Nonnull Class<T> clazz,
+ @Nonnull ObjectInspector[] ois, @Nonnull Object[] row)
+ throws HiveException, IOException {
+ final T udf;
+ try {
+ udf = clazz.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new HiveException(e);
+ }
+
+ udf.initialize(ois);
+
+ // serialization after initialization
+ byte[] serialized = serializeObjectByKryo(udf);
+ deserializeObjectByKryo(serialized, clazz);
+
+ int size = row.length;
+ GenericUDF.DeferredObject[] rowDeferred = new GenericUDF.DeferredObject[size];
+ for (int i = 0; i < size; i++) {
+ rowDeferred[i] = new GenericUDF.DeferredJavaObject(row[i]);
+ }
+
+ udf.evaluate(rowDeferred);
+
+ // serialization after evaluating row
+ serialized = serializeObjectByKryo(udf);
+ TestUtils.deserializeObjectByKryo(serialized, clazz);
+
+ udf.close();
+ }
+
+ @SuppressWarnings("deprecation")
+ public static <T extends GenericUDTF> void testGenericUDTFSerialization(@Nonnull Class<T> clazz,
+ @Nonnull ObjectInspector[] ois, @Nonnull Object[][] rows) throws HiveException {
+ final T udtf;
+ try {
+ udtf = clazz.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new HiveException(e);
+ }
+
+ udtf.initialize(ois);
+
+ // serialization after initialization
+ byte[] serialized = serializeObjectByKryo(udtf);
+ deserializeObjectByKryo(serialized, clazz);
+
+ udtf.setCollector(new Collector() {
+ public void collect(Object input) throws HiveException {
+ // noop
+ }
+ });
+
+ for (Object[] row : rows) {
+ udtf.process(row);
+ }
+
+ // serialization after processing row
+ serialized = serializeObjectByKryo(udtf);
+ TestUtils.deserializeObjectByKryo(serialized, clazz);
+
+ udtf.close();
+ }
+
+ @Nonnull
+ public static byte[] serializeObjectByKryo(@Nonnull Object obj) {
+ Kryo kryo = getKryo();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ Output output = new Output(bos);
+ kryo.writeObject(output, obj);
+ output.close();
+ return bos.toByteArray();
+ }
+
+ @Nonnull
+ public static <T> T deserializeObjectByKryo(@Nonnull byte[] in, @Nonnull Class<T> clazz) {
+ Kryo kryo = getKryo();
+ Input inp = new Input(in);
+ T t = kryo.readObject(inp, clazz);
+ inp.close();
+ return t;
+ }
+
+ @Nonnull
+ private static Kryo getKryo() {
+ return Utilities.runtimeSerializationKryo.get();
+ }
+
+}
diff --git a/xgboost/src/test/java/hivemall/xgboost/XGBoostTrainUDTFTest.java b/xgboost/src/test/java/hivemall/xgboost/XGBoostTrainUDTFTest.java
new file mode 100644
index 0000000..0d9dcfd
--- /dev/null
+++ b/xgboost/src/test/java/hivemall/xgboost/XGBoostTrainUDTFTest.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.xgboost;
+
+import biz.k11i.xgboost.Predictor;
+import biz.k11i.xgboost.util.FVec;
+import hivemall.TestBase;
+import hivemall.TestUtils;
+import hivemall.utils.lang.PrivilegedAccessor;
+import hivemall.utils.lang.mutable.MutableObject;
+import hivemall.utils.math.MathUtils;
+import hivemall.xgboost.utils.XGBoostUtils;
+import ml.dmlc.xgboost4j.java.Booster;
+import ml.dmlc.xgboost4j.java.DMatrix;
+import ml.dmlc.xgboost4j.java.XGBoostError;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.Collector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.theories.DataPoints;
+import org.junit.experimental.theories.FromDataPoints;
+import org.junit.experimental.theories.Theories;
+import org.junit.experimental.theories.Theory;
+import org.junit.runner.RunWith;
+
+@RunWith(Theories.class)
+public class XGBoostTrainUDTFTest extends TestBase {
+
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(XGBoostTrainUDTF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-objective reg:linear")},
+ new Object[][] {{Arrays.asList("1:-2", "2:-1"), 0.d}});
+ }
+
+ @DataPoints("trial")
+ public static final List<TestParameter> trial = TestParameter.merge(
+ // 1. binary classification
+ // mashroom dataset
+ new TestParameter.Builder().trainDataset(new LibsvmDataset(
+ "https://raw.githubusercontent.com/dmlc/xgboost/master/demo/data/agaricus.txt.train"))
+ .testDatase(new LibsvmDataset(
+ "https://raw.githubusercontent.com/dmlc/xgboost/master/demo/data/agaricus.txt.test"))
+ .metric(new ClassificationError(0.1f))
+ .hyperParams(new String[] {
+ "-objective binary:logistic -iters 10",
+ "-objective binary:logistic -iters 10 -num_early_stopping_rounds 3"}),
+ // 2. multiclass classification
+ // https://archive.ics.uci.edu/ml/machine-learning-databases/dermatology/dermatology.data
+ new TestParameter.Builder().trainDataset(new DermatologyDataset(true, 0.7f))
+ .testDatase(new DermatologyDataset(false, 0.7f))
+ .metric(new MultiClassClassificationError(0.1f))
+ .hyperParams(new String[] {
+ "-objective multi:softmax -num_class 6 -max_depth 6 -eta 0.1 -num_round 5"}),
+ new TestParameter.Builder().trainDataset(new DermatologyDataset(true, 0.7f))
+ .testDatase(new DermatologyDataset(false, 0.7f))
+ .metric(new MultiClassClassificationError(0.1f))
+ .hyperParams(new String[] {
+ "-objective multi:softprob -num_class 6 -max_depth 6 -eta 0.1 -num_round 5"}),
+ // 3. regression
+ // https://archive.ics.uci.edu/ml/datasets/Computer+Hardware
+ // https://github.com/dmlc/xgboost/blob/master/demo/regression/machine.conf
+ new TestParameter.Builder().trainDataset(new LibsvmDataset(
+ "https://raw.githubusercontent.com/myui/ml_dataset/master/regr/computer_hardware/machine.txt.train"))
+ .testDatase(new LibsvmDataset(
+ "https://raw.githubusercontent.com/myui/ml_dataset/master/regr/computer_hardware/machine.txt.test"))
+ .metric(new MAE(40f))
+ .hyperParams(new String[] {
+ "-booster gbtree -objective reg:linear -eta 1.0 -gamma 1.0 -min_child_weight 1.0 -max_depth 3 -num_round 5"}));
+
+
+ @Theory
+ public void testHyperParams(@FromDataPoints("trial") final TestParameter trial)
+ throws Exception {
+ final Dataset trainDataset = trial.trainDataset;
+ final Dataset testDataset = trial.testDataset;
+ final DMatrix testMatrix = testDataset.loadDatasetAsDMatrix();
+ final float[] testLabels = testMatrix.getLabel();
+ final EvalMetric metric = trial.evalMetric;
+
+ final MutableObject<float[][]> expectedPredictData = new MutableObject<>();
+ final XGBoostTrainUDTF udtf = new XGBoostTrainUDTF() {
+ @Override
+ protected void onFinishTraining(Booster booster) {
+ final float[][] result;
+ try {
+ result = booster.predict(testMatrix);
+ } catch (XGBoostError e) {
+ throw new RuntimeException(e);
+ }
+ expectedPredictData.set(result);
+ }
+ };
+ if (trainDataset.isSparseDataset()) {
+ udtf.initialize(new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ trial.hyperParams)});
+
+ } else {
+ udtf.initialize(new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaFloatObjectInspector),
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ trial.hyperParams)});
+ }
+
+ for (Object[] row : trainDataset.loadDatasetAsListOfObjects()) {
+ udtf.process(row);
+ }
+
+ udtf.setCollector(new Collector() {
+ @Override
+ public void collect(Object input) throws HiveException {
+ final float[][] expecteds = expectedPredictData.get();
+
+ Object[] forwardedObj = (Object[]) input;
+ String modelId = (String) forwardedObj[0];
+ Assert.assertNotNull(modelId);
+ Text modelStr = (Text) forwardedObj[1];
+ Booster booster = XGBoostUtils.deserializeBooster(modelStr);
+ try {
+ float[][] actuals = booster.predict(testMatrix);
+ Assert.assertEquals(expecteds.length, actuals.length);
+ for (int i = 0; i > expecteds.length; i++) {
+ Assert.assertArrayEquals(expecteds[i], actuals[i], 1e-5f);
+ }
+ } catch (XGBoostError e) {
+ throw new HiveException(e);
+ } finally {
+ XGBoostUtils.close(booster);
+ }
+
+ Predictor predictor = XGBoostUtils.loadPredictor(modelStr);
+ final String gbmName, objName;
+ try {
+ gbmName = (String) PrivilegedAccessor.getValue(predictor, "name_gbm");
+ objName = (String) PrivilegedAccessor.getValue(predictor, "name_obj");
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
+ Assert.assertEquals(udtf.params.get("booster"), gbmName);
+ Assert.assertEquals(udtf.params.get("objective"), objName);
+
+ final List<FVec> fvList;
+ try {
+ fvList = testDataset.loadDatasetAsListOfFVec();
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
+ Assert.assertEquals(expecteds.length, fvList.size());
+ int mismatches = 0;
+ for (int i = 0; i < expecteds.length; i++) {
+ float[] expected = expecteds[i];
+ FVec fv = fvList.get(i);
+ double[] actual = predictor.predict(fv);
+ Assert.assertEquals(expected.length, actual.length);
+ if (!objName.startsWith("reg:")) {
+ for (int j = 0; j < expected.length; j++) {
+ if (!MathUtils.equals(expected[j], actual[j], 1e-5d)) {
+ mismatches++;
+ break;
+ }
+ }
+ }
+ metric.next(actual, testLabels[i]);
+ }
+ Assert.assertTrue(
+ "Too many mismatches in prediction result between xgboost4j and xgboost-predictor: "
+ + mismatches,
+ mismatches <= 2);
+ }
+ });
+ udtf.close();
+ testMatrix.dispose();
+
+ metric.assertExpected();
+ }
+
+ @Test(expected = UDFArgumentException.class)
+ public void testNoObjective() throws HiveException {
+ XGBoostTrainUDTF udtf = new XGBoostTrainUDTF();
+ udtf.initialize(
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaFloatObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-num_class 4")});
+ }
+
+
+ //---------------------------------------------------
+ // multiclass target value tests
+
+ @Test
+ public void testCheckTargetValueSucess() throws HiveException {
+ XGBoostTrainUDTF udtf = new XGBoostTrainUDTF();
+ udtf.initialize(new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaFloatObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-objective multi:softmax -num_class 4")});
+
+ udtf.processTargetValue(1.0f);
+ udtf.processTargetValue(3f);
+ }
+
+ @Test(expected = UDFArgumentException.class)
+ public void testCheckInvalidTargetValue1() throws HiveException {
+ XGBoostTrainUDTF udtf = new XGBoostTrainUDTF();
+ udtf.initialize(new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaFloatObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-objective multi:softmax")});
+
+ udtf.processTargetValue(1.1f);
+ Assert.fail("-num_class option is missing");
+ }
+
+ @Test(expected = UDFArgumentException.class)
+ public void testCheckInvalidTargetValue2() throws HiveException {
+ XGBoostTrainUDTF udtf = new XGBoostTrainUDTF();
+ udtf.processOptions(new ObjectInspector[] {null, null,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-objective multi:softmax -num_class 3")});
+
+ udtf.processTargetValue(-2f);
+ Assert.fail();
+ }
+
+ @Test(expected = UDFArgumentException.class)
+ public void testCheckInvalidTargetValue3() throws HiveException {
+ XGBoostTrainUDTF udtf = new XGBoostTrainUDTF();
+ udtf.processOptions(new ObjectInspector[] {null, null,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-objective multi:softmax -num_class 3")});
+
+ udtf.processTargetValue(3f);
+ Assert.fail();
+ }
+
+}
diff --git a/xgboost/src/test/java/hivemall/xgboost/utils/DMatrixBuilderTest.java b/xgboost/src/test/java/hivemall/xgboost/utils/DMatrixBuilderTest.java
new file mode 100644
index 0000000..788693e
--- /dev/null
+++ b/xgboost/src/test/java/hivemall/xgboost/utils/DMatrixBuilderTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.xgboost.utils;
+
+import ml.dmlc.xgboost4j.java.DMatrix;
+import ml.dmlc.xgboost4j.java.XGBoostError;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class DMatrixBuilderTest {
+
+ @Test
+ public void testDenseMatrix() throws XGBoostError {
+ DMatrix matrix = createDenseDMatrix();
+ Assert.assertEquals(6, matrix.rowNum());
+ matrix.dispose();
+ }
+
+ @Test
+ public void testSparseMatrix() throws XGBoostError {
+ DMatrix matrix = createSparseDMatrix();
+ Assert.assertEquals(6, matrix.rowNum());
+ matrix.dispose();
+ }
+
+ @Test
+ public void testCreateFromCSREx() throws XGBoostError {
+ // sparse matrix
+ // 1 0 2 3 0
+ // 4 0 2 3 5
+ // 3 1 2 5 0
+ DenseDMatrixBuilder builder = new DenseDMatrixBuilder(1024);
+ builder.nextRow(new float[] {1, 0, 2, 3, 0});
+ builder.nextRow(new float[] {4, 0, 2, 3, 5});
+ builder.nextRow(new float[] {3, 1, 2, 5, 0});
+ float[] label1 = new float[] {1, 0, 1};
+ DMatrix dmat1 = builder.buildMatrix(label1);
+
+ Assert.assertEquals(3, dmat1.rowNum());
+ float[] label2 = dmat1.getLabel();
+ Assert.assertArrayEquals(label1, label2, 0.f);
+ }
+
+ private static DMatrix createDenseDMatrix() throws XGBoostError {
+ /*
+ 11 12 13 14 0 0
+ 0 22 23 0 0 0
+ 0 0 33 34 35 36
+ 0 0 0 44 45 0
+ 0 0 0 0 0 56
+ 0 0 0 0 0 66
+ */
+ DenseDMatrixBuilder builder = new DenseDMatrixBuilder(1024);
+ builder.nextRow(new String[] {"0:11", "1:12", "2:13", "3:14"});
+ builder.nextRow(new String[] {"1:22", "2:23"});
+ builder.nextRow(new String[] {"2:33", "3:34", "4:35", "5:36"});
+ builder.nextRow(new String[] {"3:44", "4:45"});
+ builder.nextRow(new String[] {"5:56"});
+ builder.nextRow(new String[] {"5:66"});
+
+ float[] labels = new float[6];
+ return builder.buildMatrix(labels);
+ }
+
+ private static DMatrix createSparseDMatrix() throws XGBoostError {
+ /*
+ 11 12 13 14 0 0
+ 0 22 23 0 0 0
+ 0 0 33 34 35 36
+ 0 0 0 44 45 0
+ 0 0 0 0 0 56
+ 0 0 0 0 0 66
+ */
+ SparseDMatrixBuilder builder = new SparseDMatrixBuilder(1024);
+ builder.nextRow(new String[] {"0:11", "1:12", "2:13", "3:14"});
+ builder.nextRow(new String[] {"1:22", "2:23"});
+ builder.nextRow(new String[] {"2:33", "3:34", "4:35", "5:36"});
+ builder.nextRow(new String[] {"3:44", "4:45"});
+ builder.nextRow(new String[] {"5:56"});
+ builder.nextRow(new String[] {"5:66"});
+
+ float[] labels = new float[6];
+ return builder.buildMatrix(labels);
+ }
+
+}