You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2019/11/27 07:12:03 UTC
[incubator-hivemall] branch master updated: [HIVEMALL-285] Add
-inspect_opts option to show hyperparameters
This is an automated email from the ASF dual-hosted git repository.
myui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git
The following commit(s) were added to refs/heads/master by this push:
new 367f984 [HIVEMALL-285] Add -inspect_opts option to show hyperparameters
367f984 is described below
commit 367f984228b1035fcecd60ec93bee840bb0e1089
Author: Makoto Yui <my...@apache.org>
AuthorDate: Wed Nov 27 16:11:56 2019 +0900
[HIVEMALL-285] Add -inspect_opts option to show hyperparameters
## What changes were proposed in this pull request?
Add `-inspect_opts` option to show hyperparameters
## What type of PR is it?
Improvement
## What is the Jira issue?
https://issues.apache.org/jira/browse/HIVEMALL-285
## How was this patch tested?
manual tests on EMR
## How to use this feature?
```sql
select train_regressor(array(), 0, '-inspect_opts -optimizer adam -reg elasticnet');
FAILED: UDFArgumentException Inspected Optimizer options ...
{disable_cvtest=false, regularization=ElasticNet, loss_function=SquaredLoss, eps=1.0E-8, decay=0.0, iterations=10, eta0=0.1, l1_ratio=0.5, lambda=1.0E-4, eta=Invscaling, optimizer=adam, beta1=0.9, beta2=0.999, alpha=1.0, cv_rate=0.005, power_t=0.1}
```
## Checklist
- [x] Did you apply source code formatter, i.e., `./bin/format_code.sh`, for your commit?
- [x] Did you run system tests on Hive (or Spark)?
Author: Makoto Yui <my...@apache.org>
Closes #220 from myui/HIVEMALL-285.
---
.../main/java/hivemall/GeneralLearnerBaseUDTF.java | 12 +++
core/src/main/java/hivemall/UDFWithOptions.java | 3 +
core/src/main/java/hivemall/UDTFWithOptions.java | 3 +
.../main/java/hivemall/optimizer/EtaEstimator.java | 40 +++++++-
.../main/java/hivemall/optimizer/Optimizer.java | 99 +++++++++++++++++-
.../java/hivemall/optimizer/Regularization.java | 26 +++++
.../classifier/GeneralClassifierUDTFTest.java | 19 ++++
docs/gitbook/supervised_learning/prediction.md | 112 ++++++++++++++++++++-
8 files changed, 307 insertions(+), 7 deletions(-)
diff --git a/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java b/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java
index af7648e..04f65e8 100644
--- a/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java
+++ b/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java
@@ -167,6 +167,7 @@ public abstract class GeneralLearnerBaseUDTF extends LearnerBaseUDTF {
@Override
protected Options getOptions() {
Options opts = super.getOptions();
+ opts.addOption("inspect_opts", false, "Inspect Optimizer options");
opts.addOption("loss", "loss_function", true, getLossOptionDescription());
opts.addOption("iter", "iterations", true,
"The maximum number of iterations [default: 10]");
@@ -217,6 +218,17 @@ public abstract class GeneralLearnerBaseUDTF extends LearnerBaseUDTF {
OptimizerOptions.processOptions(cl, optimizerOptions);
+ if (cl != null && cl.hasOption("inspect_opts")) {
+ Optimizer optimizer = createOptimizer(optimizerOptions);
+ Map<String, Object> params = optimizer.getHyperParameters();
+ params.put("loss_function", lossFunction.getType().toString());
+ params.put("iterations", iterations);
+ params.put("disable_cvtest", conversionCheck ? false : true);
+ params.put("cv_rate", convergenceRate);
+ throw new UDFArgumentException(
+ String.format("Inspected Optimizer options ...\n%s", params.toString()));
+ }
+
return cl;
}
diff --git a/core/src/main/java/hivemall/UDFWithOptions.java b/core/src/main/java/hivemall/UDFWithOptions.java
index 4fb80a5..97b430b 100644
--- a/core/src/main/java/hivemall/UDFWithOptions.java
+++ b/core/src/main/java/hivemall/UDFWithOptions.java
@@ -127,6 +127,9 @@ public abstract class UDFWithOptions extends GenericUDF {
StringWriter sw = new StringWriter();
sw.write('\n');
if (errMsg != null) {
+ if (funcDesc != null && funcDesc.name() != null) {
+ errMsg = errMsg.replace("_FUNC_", funcDesc.name());
+ }
sw.write(errMsg);
sw.write("\n\n");
}
diff --git a/core/src/main/java/hivemall/UDTFWithOptions.java b/core/src/main/java/hivemall/UDTFWithOptions.java
index 853785b..3488796 100644
--- a/core/src/main/java/hivemall/UDTFWithOptions.java
+++ b/core/src/main/java/hivemall/UDTFWithOptions.java
@@ -132,6 +132,9 @@ public abstract class UDTFWithOptions extends GenericUDTF {
StringWriter sw = new StringWriter();
sw.write('\n');
if (errMsg != null) {
+ if (funcDesc != null && funcDesc.name() != null) {
+ errMsg = errMsg.replace("_FUNC_", funcDesc.name());
+ }
sw.write(errMsg);
sw.write("\n\n");
}
diff --git a/core/src/main/java/hivemall/optimizer/EtaEstimator.java b/core/src/main/java/hivemall/optimizer/EtaEstimator.java
index 582aeb0..3bf41b5 100644
--- a/core/src/main/java/hivemall/optimizer/EtaEstimator.java
+++ b/core/src/main/java/hivemall/optimizer/EtaEstimator.java
@@ -43,6 +43,9 @@ public abstract class EtaEstimator {
this.eta0 = eta0;
}
+ @Nonnull
+ public abstract String typeName();
+
public float eta0() {
return eta0;
}
@@ -51,12 +54,22 @@ public abstract class EtaEstimator {
public void update(@Nonnegative float multiplier) {}
+ public void getHyperParameters(@Nonnull Map<String, Object> hyperParams) {
+ hyperParams.put("eta", typeName());
+ hyperParams.put("eta0", eta0());
+ }
+
public static final class FixedEtaEstimator extends EtaEstimator {
public FixedEtaEstimator(float eta) {
super(eta);
}
+ @Nonnull
+ public String typeName() {
+ return "Fixed";
+ }
+
@Override
public float eta(long t) {
return eta0;
@@ -80,6 +93,11 @@ public abstract class EtaEstimator {
this.total_steps = total_steps;
}
+ @Nonnull
+ public String typeName() {
+ return "Simple";
+ }
+
@Override
public float eta(final long t) {
if (t > total_steps) {
@@ -94,6 +112,11 @@ public abstract class EtaEstimator {
+ ", finalEta = " + finalEta + " ]";
}
+ public void getHyperParameters(@Nonnull Map<String, Object> hyperParams) {
+ super.getHyperParameters(hyperParams);
+ hyperParams.put("total_steps", total_steps);
+ }
+
}
public static final class InvscalingEtaEstimator extends EtaEstimator {
@@ -105,6 +128,11 @@ public abstract class EtaEstimator {
this.power_t = power_t;
}
+ @Nonnull
+ public String typeName() {
+ return "Invscaling";
+ }
+
@Override
public float eta(final long t) {
return (float) (eta0 / Math.pow(t, power_t));
@@ -115,6 +143,10 @@ public abstract class EtaEstimator {
return "InvscalingEtaEstimator [ eta0 = " + eta0 + ", power_t = " + power_t + " ]";
}
+ public void getHyperParameters(@Nonnull Map<String, Object> hyperParams) {
+ super.getHyperParameters(hyperParams);
+ hyperParams.put("power_t", power_t);
+ }
}
/**
@@ -130,6 +162,11 @@ public abstract class EtaEstimator {
this.eta = eta;
}
+ @Nonnull
+ public String typeName() {
+ return "boldDriver";
+ }
+
@Override
public float eta(long t) {
return eta;
@@ -207,7 +244,8 @@ public abstract class EtaEstimator {
"-total_steps MUST be provided when `-eta simple` is specified");
}
return new SimpleEtaEstimator(eta0, t);
- } else if ("inv".equalsIgnoreCase(etaScheme) || "inverse".equalsIgnoreCase(etaScheme)) {
+ } else if ("inv".equalsIgnoreCase(etaScheme) || "inverse".equalsIgnoreCase(etaScheme)
+ || "invscaling".equalsIgnoreCase(etaScheme)) {
return new InvscalingEtaEstimator(eta0, power_t);
} else {
if (StringUtils.isNumber(etaScheme)) {
diff --git a/core/src/main/java/hivemall/optimizer/Optimizer.java b/core/src/main/java/hivemall/optimizer/Optimizer.java
index f096f88..f33e6be 100644
--- a/core/src/main/java/hivemall/optimizer/Optimizer.java
+++ b/core/src/main/java/hivemall/optimizer/Optimizer.java
@@ -33,6 +33,7 @@ import hivemall.model.WeightValue.WeightValueParamsF3;
import hivemall.utils.lang.Primitives;
import hivemall.utils.math.MathUtils;
+import java.util.HashMap;
import java.util.Map;
import javax.annotation.Nonnegative;
@@ -54,6 +55,9 @@ public interface Optimizer {
@Nonnull
String getOptimizerName();
+ @Nonnull
+ Map<String, Object> getHyperParameters();
+
@NotThreadSafe
static abstract class OptimizerBase implements Optimizer {
@@ -122,6 +126,15 @@ public interface Optimizer {
return gradient;
}
+ @Override
+ public Map<String, Object> getHyperParameters() {
+ Map<String, Object> params = new HashMap<>();
+ params.put("optimizer", getOptimizerName());
+ _eta.getHyperParameters(params);
+ _reg.getHyperParameters(params);
+ return params;
+ }
+
}
static final class SGD extends OptimizerBase {
@@ -198,6 +211,15 @@ public interface Optimizer {
return nesterov ? "nesterov" : "momentum";
}
+ @Override
+ public Map<String, Object> getHyperParameters() {
+ Map<String, Object> params = super.getHyperParameters();
+ params.put("nesterov", nesterov);
+ params.put("alpha", alpha);
+ params.put("momentum", momentum);
+ return params;
+ }
+
}
static abstract class AdaGrad extends OptimizerBase {
@@ -229,9 +251,15 @@ public interface Optimizer {
return "adagrad";
}
+ @Override
+ public Map<String, Object> getHyperParameters() {
+ Map<String, Object> params = super.getHyperParameters();
+ params.put("eps", eps);
+ params.put("scale", scale);
+ return params;
+ }
}
-
/**
* RMSprop optimizer introducing weight decay to AdaGrad.
*
@@ -275,6 +303,15 @@ public interface Optimizer {
return "rmsprop";
}
+ @Override
+ public Map<String, Object> getHyperParameters() {
+ Map<String, Object> params = super.getHyperParameters();
+ params.put("decay", decay);
+ params.put("eps", eps);
+ params.put("scale", scale);
+ return params;
+ }
+
}
/**
@@ -329,6 +366,16 @@ public interface Optimizer {
return "rmsprop_graves";
}
+ @Override
+ public Map<String, Object> getHyperParameters() {
+ Map<String, Object> params = super.getHyperParameters();
+ params.put("decay", decay);
+ params.put("alpha", alpha);
+ params.put("momentum", momentum);
+ params.put("eps", eps);
+ return params;
+ }
+
}
static abstract class AdaDelta extends OptimizerBase {
@@ -382,6 +429,15 @@ public interface Optimizer {
return "adadelta";
}
+ @Override
+ public Map<String, Object> getHyperParameters() {
+ Map<String, Object> params = super.getHyperParameters();
+ params.put("decay", decay);
+ params.put("eps", eps);
+ params.put("scale", scale);
+ return params;
+ }
+
}
/**
@@ -476,6 +532,16 @@ public interface Optimizer {
return amsgrad ? "adam-amsgrad" : "adam";
}
+ @Override
+ public Map<String, Object> getHyperParameters() {
+ Map<String, Object> params = super.getHyperParameters();
+ params.put("alpha", alpha);
+ params.put("beta1", beta1);
+ params.put("beta2", beta2);
+ params.put("eps", eps);
+ params.put("decay", decay);
+ return params;
+ }
}
/**
@@ -579,6 +645,17 @@ public interface Optimizer {
return "nadam";
}
+ @Override
+ public Map<String, Object> getHyperParameters() {
+ Map<String, Object> params = super.getHyperParameters();
+ params.put("alpha", alpha);
+ params.put("beta1", beta1);
+ params.put("beta2", beta2);
+ params.put("eps", eps);
+ params.put("decay", decay);
+ params.put("scheduleDecay", scheduleDecay);
+ return params;
+ }
}
/**
@@ -634,6 +711,13 @@ public interface Optimizer {
return "eve";
}
+ @Override
+ public Map<String, Object> getHyperParameters() {
+ Map<String, Object> params = super.getHyperParameters();
+ params.put("beta3", beta3);
+ params.put("c", c);
+ return params;
+ }
}
/**
@@ -713,6 +797,12 @@ public interface Optimizer {
return "adam_hd";
}
+ @Override
+ public Map<String, Object> getHyperParameters() {
+ Map<String, Object> params = super.getHyperParameters();
+ params.put("beta", beta);
+ return params;
+ }
}
static abstract class AdagradRDA extends OptimizerBase {
@@ -760,6 +850,13 @@ public interface Optimizer {
return "adagrad_rda";
}
+ @Override
+ public Map<String, Object> getHyperParameters() {
+ Map<String, Object> params = optimizerImpl.getHyperParameters();
+ params.put("optimizer", getOptimizerName()); // replace
+ params.put("lambda", lambda);
+ return params;
+ }
}
}
diff --git a/core/src/main/java/hivemall/optimizer/Regularization.java b/core/src/main/java/hivemall/optimizer/Regularization.java
index ce7e083..e573101 100644
--- a/core/src/main/java/hivemall/optimizer/Regularization.java
+++ b/core/src/main/java/hivemall/optimizer/Regularization.java
@@ -34,6 +34,10 @@ public abstract class Regularization {
this.lambda = Primitives.parseFloat(options.get("lambda"), DEFAULT_LAMBDA);
}
+ public void getHyperParameters(@Nonnull Map<String, Object> hyperParams) {
+ hyperParams.put("lambda", lambda);
+ }
+
public float regularize(final float weight, final float gradient) {
return gradient + lambda * getRegularizer(weight);
}
@@ -56,6 +60,11 @@ public abstract class Regularization {
return gradient;
}
+ @Override
+ public void getHyperParameters(@Nonnull Map<String, Object> hyperParams) {
+ super.getHyperParameters(hyperParams);
+ hyperParams.put("regularization", "no");
+ }
}
public static final class L1 extends Regularization {
@@ -69,6 +78,11 @@ public abstract class Regularization {
return weight > 0.f ? 1.f : -1.f;
}
+ @Override
+ public void getHyperParameters(@Nonnull Map<String, Object> hyperParams) {
+ super.getHyperParameters(hyperParams);
+ hyperParams.put("regularization", "L1");
+ }
}
public static final class L2 extends Regularization {
@@ -82,6 +96,11 @@ public abstract class Regularization {
return weight;
}
+ @Override
+ public void getHyperParameters(@Nonnull Map<String, Object> hyperParams) {
+ super.getHyperParameters(hyperParams);
+ hyperParams.put("regularization", "L2");
+ }
}
public static final class ElasticNet extends Regularization {
@@ -112,6 +131,13 @@ public abstract class Regularization {
return l1Ratio * l1.getRegularizer(weight)
+ (1.f - l1Ratio) * l2.getRegularizer(weight);
}
+
+ @Override
+ public void getHyperParameters(@Nonnull Map<String, Object> hyperParams) {
+ super.getHyperParameters(hyperParams);
+ hyperParams.put("regularization", "ElasticNet");
+ hyperParams.put("l1_ratio", l1Ratio);
+ }
}
@Nonnull
diff --git a/core/src/test/java/hivemall/classifier/GeneralClassifierUDTFTest.java b/core/src/test/java/hivemall/classifier/GeneralClassifierUDTFTest.java
index 7ba336e..39c6db1 100644
--- a/core/src/test/java/hivemall/classifier/GeneralClassifierUDTFTest.java
+++ b/core/src/test/java/hivemall/classifier/GeneralClassifierUDTFTest.java
@@ -73,6 +73,25 @@ public class GeneralClassifierUDTFTest {
udtf.initialize(new ObjectInspector[] {stringListOI, intOI, params});
}
+ @Test
+ public void testInspectOptimizerOptions() throws Exception {
+ GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();
+ ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
+ ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ListObjectInspector stringListOI =
+ ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
+ ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-opt adam -reg l1 -inspect_opts");
+
+ try {
+ udtf.initialize(new ObjectInspector[] {stringListOI, intOI, params});
+ Assert.fail("should not come here");
+ } catch (UDFArgumentException e) {
+ Assert.assertTrue(e.getMessage().contains("adam"));
+ }
+ }
+
@Test(expected = UDFArgumentException.class)
public void testUnsupportedLossFunction() throws Exception {
GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();
diff --git a/docs/gitbook/supervised_learning/prediction.md b/docs/gitbook/supervised_learning/prediction.md
index e3d26c8..d5a08cb 100644
--- a/docs/gitbook/supervised_learning/prediction.md
+++ b/docs/gitbook/supervised_learning/prediction.md
@@ -222,12 +222,114 @@ For details of available options, following queries might be helpful to list all
```sql
select train_regressor('-help');
--- v0.5.0 or before
--- select train_regressor(array(), 0, '-help');
-
select train_classifier('-help');
--- v0.5.0 or before
--- select train_classifier(array(), 0, '-help');
+```
+
+```
+SELECT train_regressor('-help');
+
+FAILED: UDFArgumentException
+train_regressor takes two or three arguments: List<Int|BigInt|Text> features, float target [, constant string options]
+
+usage: train_regressor(list<string|int|bigint> features, double label [,
+ const string options]) - Returns a relation consists of
+ <string|int|bigint feature, float weight> [-alpha <arg>] [-amsgrad]
+ [-beta <arg>] [-beta1 <arg>] [-beta2 <arg>] [-beta3 <arg>] [-c
+ <arg>] [-cv_rate <arg>] [-decay] [-dense] [-dims <arg>]
+ [-disable_cv] [-disable_halffloat] [-eps <arg>] [-eta <arg>] [-eta0
+ <arg>] [-inspect_opts] [-iter <arg>] [-iters <arg>] [-l1_ratio
+ <arg>] [-lambda <arg>] [-loss <arg>] [-mini_batch <arg>] [-mix
+ <arg>] [-mix_cancel] [-mix_session <arg>] [-mix_threshold <arg>]
+ [-opt <arg>] [-power_t <arg>] [-reg <arg>] [-rho <arg>] [-scale
+ <arg>] [-ssl] [-t <arg>]
+ -alpha <arg> Coefficient of learning rate
+ [default: 1.0
+ (adam/RMSPropGraves), 0.02
+ (AdamHD/Nesterov)]
+ -amsgrad Whether to use AMSGrad variant of
+ Adam
+ -beta <arg> Hyperparameter for tuning alpha
+ in Adam-HD [default: 1e-6f]
+ -beta1,--momentum <arg> Exponential decay rate of the
+ first order moment used in Adam
+ [default: 0.9]
+ -beta2 <arg> Exponential decay rate of the
+ second order moment used in Adam
+ [default: 0.999]
+ -beta3 <arg> Exponential decay rate of alpha
+ value [default: 0.999]
+ -c <arg> Clipping constant of alpha used
+ in Eve optimizer so that clipped
+[default: 10]
+-cv_rate,--convergence_rate <arg> Threshold to determine
+ convergence [default: 0.005]
+ -decay Weight decay rate [default: 0.0]
+ -dense,--densemodel Use dense model or not
+ -dims,--feature_dimensions <arg> The dimension of model [default:
+ 16777216 (2^24)]
+ -disable_cv,--disable_cvtest Whether to disable convergence
+ check [default: OFF]
+ -disable_halffloat Toggle this option to disable the
+ use of SpaceEfficientDenseModel
+ -eps <arg> Denominator value of
+ AdaDelta/AdaGrad/Adam [default:
+ 1e-8 (AdaDelta/Adam), 1.0
+ (Adagrad)]
+ -eta <arg> Learning rate scheme [default:
+ inverse/inv, fixed, simple]
+ -eta0 <arg> The initial learning rate
+ [default: 0.1]
+ -inspect_opts Inspect Optimizer options
+ -iter,--iterations <arg> The maximum number of iterations
+ [default: 10]
+ -iters,--iterations <arg> The maximum number of iterations
+ [default: 10]
+ -l1_ratio <arg> Ratio of L1 regularizer as a part
+ of Elastic Net regularization
+ [default: 0.5]
+ -lambda <arg> Regularization term [default
+ 0.0001]
+ -loss,--loss_function <arg> Loss function [SquaredLoss
+ (default), QuantileLoss,
+ EpsilonInsensitiveLoss,
+ SquaredEpsilonInsensitiveLoss,
+ HuberLoss]
+ -mini_batch,--mini_batch_size <arg> Mini batch size [default: 1].
+ Expecting the value in range
+ [1,100] or so.
+ -mix,--mix_servers <arg> Comma separated list of MIX
+ servers
+ -mix_cancel,--enable_mix_canceling Enable mix cancel requests
+ -mix_session,--mix_session_name <arg> Mix session name [default:
+ ${mapred.job.id}]
+ -mix_threshold <arg> Threshold to mix local updates in
+ range (0,127] [default: 3]
+ -opt,--optimizer <arg> Optimizer to update weights
+ [default: adagrad, sgd, momentum,
+ nesterov, rmsprop, rmspropgraves,
+ adadelta, adam, eve, adam_hd]
+ -power_t <arg> The exponent for inverse scaling
+ learning rate [default: 0.1]
+ -reg,--regularization <arg> Regularization type [default:
+ rda, l1, l2, elasticnet]
+ -rho,--decay <arg> Exponential decay rate of the
+ first and second order moments
+ [default 0.95 (AdaDelta,
+ rmsprop)]
+ -scale <arg> Scaling factor for cumulative
+ weights [100.0]
+ -ssl Use SSL for the communication
+ with mix servers
+ -t,--total_steps <arg> a total of n_samples * epochs
+time steps
```
In practice, you can try different combinations of the options in order to achieve higher prediction accuracy.
+
+You can also find the default optimizer hyperparameters by `-inspect_opts` option as follows:
+
+```sql
+select train_regressor(array(), 0, '-inspect_opts -optimizer adam -reg l1');
+
+FAILED: UDFArgumentException Inspected Optimizer options ...
+{disable_cvtest=false, regularization=L1, loss_function=SquaredLoss, eps=1.0E-8, decay=0.0, iterations=10, eta0=0.1, lambda=1.0E-4, eta=Invscaling, optimizer=adam, beta1=0.9, beta2=0.999, alpha=1.0, cv_rate=0.005, power_t=0.1}```