You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2016/12/02 07:04:15 UTC
[13/50] [abbrv] incubator-hivemall git commit: Add optimizer
implementations
Add optimizer implementations
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/f81948c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/f81948c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/f81948c5
Branch: refs/heads/JIRA-22/pr-285
Commit: f81948c5c7b83155eb29369a59f1fc65bb607f91
Parents: 5a7df55
Author: Takeshi YAMAMURO <li...@gmail.com>
Authored: Mon May 2 23:43:42 2016 +0900
Committer: Takeshi YAMAMURO <li...@gmail.com>
Committed: Wed Sep 21 00:07:28 2016 +0900
----------------------------------------------------------------------
.../src/main/java/hivemall/LearnerBaseUDTF.java | 22 +
.../hivemall/classifier/AROWClassifierUDTF.java | 2 +-
.../hivemall/classifier/AdaGradRDAUDTF.java | 123 +----
.../classifier/BinaryOnlineClassifierUDTF.java | 3 +
.../classifier/GeneralClassifierUDTF.java | 121 +++++
.../classifier/PassiveAggressiveUDTF.java | 2 +-
.../main/java/hivemall/common/EtaEstimator.java | 160 -------
.../java/hivemall/common/LossFunctions.java | 467 -------------------
.../java/hivemall/fm/FMHyperParameters.java | 2 +-
.../hivemall/fm/FactorizationMachineModel.java | 2 +-
.../hivemall/fm/FactorizationMachineUDTF.java | 8 +-
.../fm/FieldAwareFactorizationMachineModel.java | 1 +
.../hivemall/mf/BPRMatrixFactorizationUDTF.java | 2 +-
.../hivemall/mf/MatrixFactorizationSGDUDTF.java | 2 +-
.../main/java/hivemall/model/DenseModel.java | 87 +---
.../main/java/hivemall/model/IWeightValue.java | 16 +-
.../java/hivemall/model/PredictionModel.java | 5 +-
.../model/SpaceEfficientDenseModel.java | 93 +---
.../main/java/hivemall/model/SparseModel.java | 20 +-
.../model/SynchronizedModelWrapper.java | 16 +-
.../main/java/hivemall/model/WeightValue.java | 162 ++++++-
.../hivemall/model/WeightValueWithClock.java | 167 ++++++-
.../optimizer/DenseOptimizerFactory.java | 215 +++++++++
.../java/hivemall/optimizer/EtaEstimator.java | 191 ++++++++
.../java/hivemall/optimizer/LossFunctions.java | 467 +++++++++++++++++++
.../main/java/hivemall/optimizer/Optimizer.java | 246 ++++++++++
.../java/hivemall/optimizer/Regularization.java | 99 ++++
.../optimizer/SparseOptimizerFactory.java | 171 +++++++
.../hivemall/regression/AROWRegressionUDTF.java | 2 +-
.../java/hivemall/regression/AdaDeltaUDTF.java | 117 +----
.../java/hivemall/regression/AdaGradUDTF.java | 118 +----
.../regression/GeneralRegressionUDTF.java | 125 +++++
.../java/hivemall/regression/LogressUDTF.java | 63 +--
.../PassiveAggressiveRegressionUDTF.java | 2 +-
.../hivemall/regression/RegressionBaseUDTF.java | 14 +-
.../java/hivemall/optimizer/OptimizerTest.java | 172 +++++++
.../java/hivemall/mix/server/MixServerTest.java | 14 +-
resources/ddl/define-all-as-permanent.hive | 13 +-
resources/ddl/define-all.hive | 12 +-
39 files changed, 2301 insertions(+), 1223 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/LearnerBaseUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/LearnerBaseUDTF.java b/core/src/main/java/hivemall/LearnerBaseUDTF.java
index 4518cce..7fd5190 100644
--- a/core/src/main/java/hivemall/LearnerBaseUDTF.java
+++ b/core/src/main/java/hivemall/LearnerBaseUDTF.java
@@ -28,6 +28,9 @@ import hivemall.model.SparseModel;
import hivemall.model.SynchronizedModelWrapper;
import hivemall.model.WeightValue;
import hivemall.model.WeightValue.WeightValueWithCovar;
+import hivemall.optimizer.DenseOptimizerFactory;
+import hivemall.optimizer.Optimizer;
+import hivemall.optimizer.SparseOptimizerFactory;
import hivemall.utils.datetime.StopWatch;
import hivemall.utils.hadoop.HadoopUtils;
import hivemall.utils.hadoop.HiveUtils;
@@ -38,6 +41,7 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -195,6 +199,24 @@ public abstract class LearnerBaseUDTF extends UDTFWithOptions {
return model;
}
+ // If a model implements a optimizer, it must override this
+ protected Map<String, String> getOptimzierOptions() {
+ return null;
+ }
+
+ protected Optimizer createOptimizer() {
+ assert(!useCovariance());
+ final Map<String, String> options = getOptimzierOptions();
+ if(options != null) {
+ if (dense_model) {
+ return DenseOptimizerFactory.create(model_dims, options);
+ } else {
+ return SparseOptimizerFactory.create(model_dims, options);
+ }
+ }
+ return null;
+ }
+
protected MixClient configureMixClient(String connectURIs, String label, PredictionModel model) {
assert (connectURIs != null);
assert (model != null);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/classifier/AROWClassifierUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/classifier/AROWClassifierUDTF.java b/core/src/main/java/hivemall/classifier/AROWClassifierUDTF.java
index e5ef975..ac8afcb 100644
--- a/core/src/main/java/hivemall/classifier/AROWClassifierUDTF.java
+++ b/core/src/main/java/hivemall/classifier/AROWClassifierUDTF.java
@@ -18,7 +18,7 @@
*/
package hivemall.classifier;
-import hivemall.common.LossFunctions;
+import hivemall.optimizer.LossFunctions;
import hivemall.model.FeatureValue;
import hivemall.model.IWeightValue;
import hivemall.model.PredictionResult;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/classifier/AdaGradRDAUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/classifier/AdaGradRDAUDTF.java b/core/src/main/java/hivemall/classifier/AdaGradRDAUDTF.java
index 1351bca..a6714f4 100644
--- a/core/src/main/java/hivemall/classifier/AdaGradRDAUDTF.java
+++ b/core/src/main/java/hivemall/classifier/AdaGradRDAUDTF.java
@@ -18,124 +18,13 @@
*/
package hivemall.classifier;
-import hivemall.common.LossFunctions;
-import hivemall.model.FeatureValue;
-import hivemall.model.IWeightValue;
-import hivemall.model.WeightValue.WeightValueParamsF2;
-import hivemall.utils.lang.Primitives;
+@Deprecated
+public final class AdaGradRDAUDTF extends GeneralClassifierUDTF {
-import javax.annotation.Nonnull;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-
-@Description(name = "train_adagrad_rda",
- value = "_FUNC_(list<string|int|bigint> features, int label [, const string options])"
- + " - Returns a relation consists of <string|int|bigint feature, float weight>",
- extended = "Build a prediction model by Adagrad+RDA regularization binary classifier")
-public final class AdaGradRDAUDTF extends BinaryOnlineClassifierUDTF {
-
- private float eta;
- private float lambda;
- private float scaling;
-
- @Override
- public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
- final int numArgs = argOIs.length;
- if (numArgs != 2 && numArgs != 3) {
- throw new UDFArgumentException(
- "_FUNC_ takes 2 or 3 arguments: List<Text|Int|BitInt> features, int label [, constant string options]");
- }
-
- StructObjectInspector oi = super.initialize(argOIs);
- model.configureParams(true, false, true);
- return oi;
+ public AdaGradRDAUDTF() {
+ optimizerOptions.put("optimizer", "AdaGrad");
+ optimizerOptions.put("regularization", "RDA");
+ optimizerOptions.put("lambda", "1e-6");
}
- @Override
- protected Options getOptions() {
- Options opts = super.getOptions();
- opts.addOption("eta", "eta0", true, "The learning rate \\eta [default 0.1]");
- opts.addOption("lambda", true, "lambda constant of RDA [default: 1E-6f]");
- opts.addOption("scale", true,
- "Internal scaling/descaling factor for cumulative weights [default: 100]");
- return opts;
- }
-
- @Override
- protected CommandLine processOptions(ObjectInspector[] argOIs) throws UDFArgumentException {
- CommandLine cl = super.processOptions(argOIs);
- if (cl == null) {
- this.eta = 0.1f;
- this.lambda = 1E-6f;
- this.scaling = 100f;
- } else {
- this.eta = Primitives.parseFloat(cl.getOptionValue("eta"), 0.1f);
- this.lambda = Primitives.parseFloat(cl.getOptionValue("lambda"), 1E-6f);
- this.scaling = Primitives.parseFloat(cl.getOptionValue("scale"), 100f);
- }
- return cl;
- }
-
- @Override
- protected void train(@Nonnull final FeatureValue[] features, final int label) {
- final float y = label > 0 ? 1.f : -1.f;
-
- float p = predict(features);
- float loss = LossFunctions.hingeLoss(p, y); // 1.0 - y * p
- if (loss <= 0.f) { // max(0, 1 - y * p)
- return;
- }
- // subgradient => -y * W dot xi
- update(features, y, count);
- }
-
- protected void update(@Nonnull final FeatureValue[] features, final float y, final int t) {
- for (FeatureValue f : features) {// w[f] += y * x[f]
- if (f == null) {
- continue;
- }
- Object x = f.getFeature();
- float xi = f.getValueAsFloat();
-
- updateWeight(x, xi, y, t);
- }
- }
-
- protected void updateWeight(@Nonnull final Object x, final float xi, final float y,
- final float t) {
- final float gradient = -y * xi;
- final float scaled_gradient = gradient * scaling;
-
- float scaled_sum_sqgrad = 0.f;
- float scaled_sum_grad = 0.f;
- IWeightValue old = model.get(x);
- if (old != null) {
- scaled_sum_sqgrad = old.getSumOfSquaredGradients();
- scaled_sum_grad = old.getSumOfGradients();
- }
- scaled_sum_grad += scaled_gradient;
- scaled_sum_sqgrad += (scaled_gradient * scaled_gradient);
-
- float sum_grad = scaled_sum_grad * scaling;
- double sum_sqgrad = scaled_sum_sqgrad * scaling;
-
- // sign(u_{t,i})
- float sign = (sum_grad > 0.f) ? 1.f : -1.f;
- // |u_{t,i}|/t - \lambda
- float meansOfGradients = sign * sum_grad / t - lambda;
- if (meansOfGradients < 0.f) {
- // x_{t,i} = 0
- model.delete(x);
- } else {
- // x_{t,i} = -sign(u_{t,i}) * \frac{\eta t}{\sqrt{G_{t,ii}}}(|u_{t,i}|/t - \lambda)
- float weight = -1.f * sign * eta * t * meansOfGradients / (float) Math.sqrt(sum_sqgrad);
- IWeightValue new_w = new WeightValueParamsF2(weight, scaled_sum_sqgrad, scaled_sum_grad);
- model.set(x, new_w);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/classifier/BinaryOnlineClassifierUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/classifier/BinaryOnlineClassifierUDTF.java b/core/src/main/java/hivemall/classifier/BinaryOnlineClassifierUDTF.java
index 43a124d..0ee5d5f 100644
--- a/core/src/main/java/hivemall/classifier/BinaryOnlineClassifierUDTF.java
+++ b/core/src/main/java/hivemall/classifier/BinaryOnlineClassifierUDTF.java
@@ -25,6 +25,7 @@ import hivemall.model.PredictionModel;
import hivemall.model.PredictionResult;
import hivemall.model.WeightValue;
import hivemall.model.WeightValue.WeightValueWithCovar;
+import hivemall.optimizer.Optimizer;
import hivemall.utils.collections.IMapIterator;
import hivemall.utils.hadoop.HiveUtils;
@@ -56,6 +57,7 @@ public abstract class BinaryOnlineClassifierUDTF extends LearnerBaseUDTF {
private boolean parseFeature;
protected PredictionModel model;
+ protected Optimizer optimizerImpl;
protected int count;
@Override
@@ -76,6 +78,7 @@ public abstract class BinaryOnlineClassifierUDTF extends LearnerBaseUDTF {
if (preloadedModelFile != null) {
loadPredictionModel(model, preloadedModelFile, featureOutputOI);
}
+ this.optimizerImpl = createOptimizer();
this.count = 0;
return getReturnOI(featureOutputOI);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/classifier/GeneralClassifierUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/classifier/GeneralClassifierUDTF.java b/core/src/main/java/hivemall/classifier/GeneralClassifierUDTF.java
new file mode 100644
index 0000000..feebadd
--- /dev/null
+++ b/core/src/main/java/hivemall/classifier/GeneralClassifierUDTF.java
@@ -0,0 +1,121 @@
+/*
+ * Hivemall: Hive scalable Machine Learning Library
+ *
+ * Copyright (C) 2015 Makoto YUI
+ * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hivemall.classifier;
+
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.Nonnull;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+
+import hivemall.optimizer.LossFunctions;
+import hivemall.model.FeatureValue;
+
+/**
+ * A general classifier class with replaceable optimization functions.
+ */
+public class GeneralClassifierUDTF extends BinaryOnlineClassifierUDTF {
+
+ protected final Map<String, String> optimizerOptions;
+
+ public GeneralClassifierUDTF() {
+ this.optimizerOptions = new HashMap<String, String>();
+ // Set default values
+ optimizerOptions.put("optimizer", "adagrad");
+ optimizerOptions.put("eta", "fixed");
+ optimizerOptions.put("eta0", "1.0");
+ optimizerOptions.put("regularization", "RDA");
+ optimizerOptions.put("lambda", "1e-6");
+ optimizerOptions.put("scale", "100.0");
+ optimizerOptions.put("lambda", "1.0");
+ }
+
+ @Override
+ public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
+ if(argOIs.length != 2 && argOIs.length != 3) {
+ throw new UDFArgumentException(
+ this.getClass().getSimpleName()
+ + " takes 2 or 3 arguments: List<Text|Int|BitInt> features, int label "
+ + "[, constant string options]");
+ }
+ return super.initialize(argOIs);
+ }
+
+ @Override
+ protected Options getOptions() {
+ Options opts = super.getOptions();
+ opts.addOption("optimizer", "opt", true, "Optimizer to update weights [default: adagrad+rda]");
+ opts.addOption("eta", "eta0", true, "Initial learning rate [default 1.0]");
+ opts.addOption("lambda", true, "Lambda value of RDA [default: 1e-6f]");
+ opts.addOption("scale", true, "Scaling factor for cumulative weights [100.0]");
+ opts.addOption("regularization", "reg", true, "Regularization type [default not-defined]");
+ opts.addOption("lambda", true, "Regularization term on weights [default 1.0]");
+ return opts;
+ }
+
+ @Override
+ protected CommandLine processOptions(ObjectInspector[] argOIs) throws UDFArgumentException {
+ final CommandLine cl = super.processOptions(argOIs);
+ assert(cl != null);
+ if(cl != null) {
+ for(final String arg : cl.getArgs()) {
+ optimizerOptions.put(arg, cl.getOptionValue(arg));
+ }
+ }
+ return cl;
+ }
+
+ @Override
+ protected Map<String, String> getOptimzierOptions() {
+ return optimizerOptions;
+ }
+
+ @Override
+ protected void train(@Nonnull final FeatureValue[] features, final int label) {
+ float predicted = predict(features);
+ update(features, label > 0 ? 1.f : -1.f, predicted);
+ }
+
+ @Override
+ protected void update(@Nonnull final FeatureValue[] features, final float label,
+ final float predicted) {
+ if(is_mini_batch) {
+ throw new UnsupportedOperationException(
+ this.getClass().getSimpleName() + " supports no `is_mini_batch` mode");
+ } else {
+ float loss = LossFunctions.hingeLoss(predicted, label);
+ if(loss <= 0.f) {
+ return;
+ }
+ for(FeatureValue f : features) {
+ Object feature = f.getFeature();
+ float xi = f.getValueAsFloat();
+ float weight = model.getWeight(feature);
+ float new_weight = optimizerImpl.computeUpdatedValue(feature, weight, -label * xi);
+ model.setWeight(feature, new_weight);
+ }
+ optimizerImpl.proceedStep();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/classifier/PassiveAggressiveUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/classifier/PassiveAggressiveUDTF.java b/core/src/main/java/hivemall/classifier/PassiveAggressiveUDTF.java
index 0213dec..9e404cd 100644
--- a/core/src/main/java/hivemall/classifier/PassiveAggressiveUDTF.java
+++ b/core/src/main/java/hivemall/classifier/PassiveAggressiveUDTF.java
@@ -18,7 +18,7 @@
*/
package hivemall.classifier;
-import hivemall.common.LossFunctions;
+import hivemall.optimizer.LossFunctions;
import hivemall.model.FeatureValue;
import hivemall.model.PredictionResult;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/common/EtaEstimator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/common/EtaEstimator.java b/core/src/main/java/hivemall/common/EtaEstimator.java
deleted file mode 100644
index 3287641..0000000
--- a/core/src/main/java/hivemall/common/EtaEstimator.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Hivemall: Hive scalable Machine Learning Library
- *
- * Copyright (C) 2015 Makoto YUI
- * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hivemall.common;
-
-import hivemall.utils.lang.NumberUtils;
-import hivemall.utils.lang.Primitives;
-
-import javax.annotation.Nonnegative;
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
-
-public abstract class EtaEstimator {
-
- protected final float eta0;
-
- public EtaEstimator(float eta0) {
- this.eta0 = eta0;
- }
-
- public float eta0() {
- return eta0;
- }
-
- public abstract float eta(long t);
-
- public void update(@Nonnegative float multipler) {}
-
- public static final class FixedEtaEstimator extends EtaEstimator {
-
- public FixedEtaEstimator(float eta) {
- super(eta);
- }
-
- @Override
- public float eta(long t) {
- return eta0;
- }
-
- }
-
- public static final class SimpleEtaEstimator extends EtaEstimator {
-
- private final float finalEta;
- private final double total_steps;
-
- public SimpleEtaEstimator(float eta0, long total_steps) {
- super(eta0);
- this.finalEta = (float) (eta0 / 2.d);
- this.total_steps = total_steps;
- }
-
- @Override
- public float eta(final long t) {
- if (t > total_steps) {
- return finalEta;
- }
- return (float) (eta0 / (1.d + (t / total_steps)));
- }
-
- }
-
- public static final class InvscalingEtaEstimator extends EtaEstimator {
-
- private final double power_t;
-
- public InvscalingEtaEstimator(float eta0, double power_t) {
- super(eta0);
- this.power_t = power_t;
- }
-
- @Override
- public float eta(final long t) {
- return (float) (eta0 / Math.pow(t, power_t));
- }
-
- }
-
- /**
- * bold driver: Gemulla et al., Large-scale matrix factorization with distributed stochastic
- * gradient descent, KDD 2011.
- */
- public static final class AdjustingEtaEstimator extends EtaEstimator {
-
- private float eta;
-
- public AdjustingEtaEstimator(float eta) {
- super(eta);
- this.eta = eta;
- }
-
- @Override
- public float eta(long t) {
- return eta;
- }
-
- @Override
- public void update(@Nonnegative float multipler) {
- float newEta = eta * multipler;
- if (!NumberUtils.isFinite(newEta)) {
- // avoid NaN or INFINITY
- return;
- }
- this.eta = Math.min(eta0, newEta); // never be larger than eta0
- }
-
- }
-
- @Nonnull
- public static EtaEstimator get(@Nullable CommandLine cl) throws UDFArgumentException {
- return get(cl, 0.1f);
- }
-
- @Nonnull
- public static EtaEstimator get(@Nullable CommandLine cl, float defaultEta0)
- throws UDFArgumentException {
- if (cl == null) {
- return new InvscalingEtaEstimator(defaultEta0, 0.1d);
- }
-
- if (cl.hasOption("boldDriver")) {
- float eta = Primitives.parseFloat(cl.getOptionValue("eta"), 0.3f);
- return new AdjustingEtaEstimator(eta);
- }
-
- String etaValue = cl.getOptionValue("eta");
- if (etaValue != null) {
- float eta = Float.parseFloat(etaValue);
- return new FixedEtaEstimator(eta);
- }
-
- float eta0 = Primitives.parseFloat(cl.getOptionValue("eta0"), defaultEta0);
- if (cl.hasOption("t")) {
- long t = Long.parseLong(cl.getOptionValue("t"));
- return new SimpleEtaEstimator(eta0, t);
- }
-
- double power_t = Primitives.parseDouble(cl.getOptionValue("power_t"), 0.1d);
- return new InvscalingEtaEstimator(eta0, power_t);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/common/LossFunctions.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/common/LossFunctions.java b/core/src/main/java/hivemall/common/LossFunctions.java
deleted file mode 100644
index 6b403fd..0000000
--- a/core/src/main/java/hivemall/common/LossFunctions.java
+++ /dev/null
@@ -1,467 +0,0 @@
-/*
- * Hivemall: Hive scalable Machine Learning Library
- *
- * Copyright (C) 2015 Makoto YUI
- * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hivemall.common;
-
-import hivemall.utils.math.MathUtils;
-
-/**
- * @link https://github.com/JohnLangford/vowpal_wabbit/wiki/Loss-functions
- */
-public final class LossFunctions {
-
- public enum LossType {
- SquaredLoss, LogLoss, HingeLoss, SquaredHingeLoss, QuantileLoss, EpsilonInsensitiveLoss
- }
-
- public static LossFunction getLossFunction(String type) {
- if ("SquaredLoss".equalsIgnoreCase(type)) {
- return new SquaredLoss();
- } else if ("LogLoss".equalsIgnoreCase(type)) {
- return new LogLoss();
- } else if ("HingeLoss".equalsIgnoreCase(type)) {
- return new HingeLoss();
- } else if ("SquaredHingeLoss".equalsIgnoreCase(type)) {
- return new SquaredHingeLoss();
- } else if ("QuantileLoss".equalsIgnoreCase(type)) {
- return new QuantileLoss();
- } else if ("EpsilonInsensitiveLoss".equalsIgnoreCase(type)) {
- return new EpsilonInsensitiveLoss();
- }
- throw new IllegalArgumentException("Unsupported type: " + type);
- }
-
- public static LossFunction getLossFunction(LossType type) {
- switch (type) {
- case SquaredLoss:
- return new SquaredLoss();
- case LogLoss:
- return new LogLoss();
- case HingeLoss:
- return new HingeLoss();
- case SquaredHingeLoss:
- return new SquaredHingeLoss();
- case QuantileLoss:
- return new QuantileLoss();
- case EpsilonInsensitiveLoss:
- return new EpsilonInsensitiveLoss();
- default:
- throw new IllegalArgumentException("Unsupported type: " + type);
- }
- }
-
- public interface LossFunction {
-
- /**
- * Evaluate the loss function.
- *
- * @param p The prediction, p = w^T x
- * @param y The true value (aka target)
- * @return The loss evaluated at `p` and `y`.
- */
- public float loss(float p, float y);
-
- public double loss(double p, double y);
-
- /**
- * Evaluate the derivative of the loss function with respect to the prediction `p`.
- *
- * @param p The prediction, p = w^T x
- * @param y The true value (aka target)
- * @return The derivative of the loss function w.r.t. `p`.
- */
- public float dloss(float p, float y);
-
- public boolean forBinaryClassification();
-
- public boolean forRegression();
-
- }
-
- public static abstract class BinaryLoss implements LossFunction {
-
- protected static void checkTarget(float y) {
- if (!(y == 1.f || y == -1.f)) {
- throw new IllegalArgumentException("target must be [+1,-1]: " + y);
- }
- }
-
- protected static void checkTarget(double y) {
- if (!(y == 1.d || y == -1.d)) {
- throw new IllegalArgumentException("target must be [+1,-1]: " + y);
- }
- }
-
- @Override
- public boolean forBinaryClassification() {
- return true;
- }
-
- @Override
- public boolean forRegression() {
- return false;
- }
- }
-
- public static abstract class RegressionLoss implements LossFunction {
-
- @Override
- public boolean forBinaryClassification() {
- return false;
- }
-
- @Override
- public boolean forRegression() {
- return true;
- }
-
- }
-
- /**
- * Squared loss for regression problems.
- *
- * If you're trying to minimize the mean error, use squared-loss.
- */
- public static final class SquaredLoss extends RegressionLoss {
-
- @Override
- public float loss(float p, float y) {
- final float z = p - y;
- return z * z * 0.5f;
- }
-
- @Override
- public double loss(double p, double y) {
- final double z = p - y;
- return z * z * 0.5d;
- }
-
- @Override
- public float dloss(float p, float y) {
- return p - y; // 2 (p - y) / 2
- }
- }
-
- /**
- * Logistic regression loss for binary classification with y in {-1, 1}.
- */
- public static final class LogLoss extends BinaryLoss {
-
- /**
- * <code>logloss(p,y) = log(1+exp(-p*y))</code>
- */
- @Override
- public float loss(float p, float y) {
- checkTarget(y);
-
- final float z = y * p;
- if (z > 18.f) {
- return (float) Math.exp(-z);
- }
- if (z < -18.f) {
- return -z;
- }
- return (float) Math.log(1.d + Math.exp(-z));
- }
-
- @Override
- public double loss(double p, double y) {
- checkTarget(y);
-
- final double z = y * p;
- if (z > 18.d) {
- return Math.exp(-z);
- }
- if (z < -18.d) {
- return -z;
- }
- return Math.log(1.d + Math.exp(-z));
- }
-
- @Override
- public float dloss(float p, float y) {
- checkTarget(y);
-
- float z = y * p;
- if (z > 18.f) {
- return (float) Math.exp(-z) * -y;
- }
- if (z < -18.f) {
- return -y;
- }
- return -y / ((float) Math.exp(z) + 1.f);
- }
- }
-
- /**
- * Hinge loss for binary classification tasks with y in {-1,1}.
- */
- public static final class HingeLoss extends BinaryLoss {
-
- private float threshold;
-
- public HingeLoss() {
- this(1.f);
- }
-
- /**
- * @param threshold Margin threshold. When threshold=1.0, one gets the loss used by SVM.
- * When threshold=0.0, one gets the loss used by the Perceptron.
- */
- public HingeLoss(float threshold) {
- this.threshold = threshold;
- }
-
- public void setThreshold(float threshold) {
- this.threshold = threshold;
- }
-
- @Override
- public float loss(float p, float y) {
- float loss = hingeLoss(p, y, threshold);
- return (loss > 0.f) ? loss : 0.f;
- }
-
- @Override
- public double loss(double p, double y) {
- double loss = hingeLoss(p, y, threshold);
- return (loss > 0.d) ? loss : 0.d;
- }
-
- @Override
- public float dloss(float p, float y) {
- float loss = hingeLoss(p, y, threshold);
- return (loss > 0.f) ? -y : 0.f;
- }
- }
-
- /**
- * Squared Hinge loss for binary classification tasks with y in {-1,1}.
- */
- public static final class SquaredHingeLoss extends BinaryLoss {
-
- @Override
- public float loss(float p, float y) {
- return squaredHingeLoss(p, y);
- }
-
- @Override
- public double loss(double p, double y) {
- return squaredHingeLoss(p, y);
- }
-
- @Override
- public float dloss(float p, float y) {
- checkTarget(y);
-
- float d = 1 - (y * p);
- return (d > 0.f) ? -2.f * d * y : 0.f;
- }
-
- }
-
- /**
- * Quantile loss is useful to predict rank/order and you do not mind the mean error to increase
- * as long as you get the relative order correct.
- *
- * @link http://en.wikipedia.org/wiki/Quantile_regression
- */
- public static final class QuantileLoss extends RegressionLoss {
-
- private float tau;
-
- public QuantileLoss() {
- this.tau = 0.5f;
- }
-
- public QuantileLoss(float tau) {
- setTau(tau);
- }
-
- public void setTau(float tau) {
- if (tau <= 0 || tau >= 1.0) {
- throw new IllegalArgumentException("tau must be in range (0, 1): " + tau);
- }
- this.tau = tau;
- }
-
- @Override
- public float loss(float p, float y) {
- float e = y - p;
- if (e > 0.f) {
- return tau * e;
- } else {
- return -(1.f - tau) * e;
- }
- }
-
- @Override
- public double loss(double p, double y) {
- double e = y - p;
- if (e > 0.d) {
- return tau * e;
- } else {
- return -(1.d - tau) * e;
- }
- }
-
- @Override
- public float dloss(float p, float y) {
- float e = y - p;
- if (e == 0.f) {
- return 0.f;
- }
- return (e > 0.f) ? -tau : (1.f - tau);
- }
-
- }
-
- /**
- * Epsilon-Insensitive loss used by Support Vector Regression (SVR).
- * <code>loss = max(0, |y - p| - epsilon)</code>
- */
- public static final class EpsilonInsensitiveLoss extends RegressionLoss {
-
- private float epsilon;
-
- public EpsilonInsensitiveLoss() {
- this(0.1f);
- }
-
- public EpsilonInsensitiveLoss(float epsilon) {
- this.epsilon = epsilon;
- }
-
- public void setEpsilon(float epsilon) {
- this.epsilon = epsilon;
- }
-
- @Override
- public float loss(float p, float y) {
- float loss = Math.abs(y - p) - epsilon;
- return (loss > 0.f) ? loss : 0.f;
- }
-
- @Override
- public double loss(double p, double y) {
- double loss = Math.abs(y - p) - epsilon;
- return (loss > 0.d) ? loss : 0.d;
- }
-
- @Override
- public float dloss(float p, float y) {
- if ((y - p) > epsilon) {// real value > predicted value - epsilon
- return -1.f;
- }
- if ((p - y) > epsilon) {// real value < predicted value - epsilon
- return 1.f;
- }
- return 0.f;
- }
-
- }
-
- public static float logisticLoss(final float target, final float predicted) {
- if (predicted > -100.d) {
- return target - (float) MathUtils.sigmoid(predicted);
- } else {
- return target;
- }
- }
-
- public static float logLoss(final float p, final float y) {
- BinaryLoss.checkTarget(y);
-
- final float z = y * p;
- if (z > 18.f) {
- return (float) Math.exp(-z);
- }
- if (z < -18.f) {
- return -z;
- }
- return (float) Math.log(1.d + Math.exp(-z));
- }
-
- public static double logLoss(final double p, final double y) {
- BinaryLoss.checkTarget(y);
-
- final double z = y * p;
- if (z > 18.d) {
- return Math.exp(-z);
- }
- if (z < -18.d) {
- return -z;
- }
- return Math.log(1.d + Math.exp(-z));
- }
-
- public static float squaredLoss(float p, float y) {
- final float z = p - y;
- return z * z * 0.5f;
- }
-
- public static double squaredLoss(double p, double y) {
- final double z = p - y;
- return z * z * 0.5d;
- }
-
- public static float hingeLoss(final float p, final float y, final float threshold) {
- BinaryLoss.checkTarget(y);
-
- float z = y * p;
- return threshold - z;
- }
-
- public static double hingeLoss(final double p, final double y, final double threshold) {
- BinaryLoss.checkTarget(y);
-
- double z = y * p;
- return threshold - z;
- }
-
- public static float hingeLoss(float p, float y) {
- return hingeLoss(p, y, 1.f);
- }
-
- public static double hingeLoss(double p, double y) {
- return hingeLoss(p, y, 1.d);
- }
-
- public static float squaredHingeLoss(final float p, final float y) {
- BinaryLoss.checkTarget(y);
-
- float z = y * p;
- float d = 1.f - z;
- return (d > 0.f) ? (d * d) : 0.f;
- }
-
- public static double squaredHingeLoss(final double p, final double y) {
- BinaryLoss.checkTarget(y);
-
- double z = y * p;
- double d = 1.d - z;
- return (d > 0.d) ? d * d : 0.d;
- }
-
- /**
- * Math.abs(target - predicted) - epsilon
- */
- public static float epsilonInsensitiveLoss(float predicted, float target, float epsilon) {
- return Math.abs(target - predicted) - epsilon;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/fm/FMHyperParameters.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/fm/FMHyperParameters.java b/core/src/main/java/hivemall/fm/FMHyperParameters.java
index db69db3..512476d 100644
--- a/core/src/main/java/hivemall/fm/FMHyperParameters.java
+++ b/core/src/main/java/hivemall/fm/FMHyperParameters.java
@@ -17,8 +17,8 @@
*/
package hivemall.fm;
-import hivemall.common.EtaEstimator;
import hivemall.fm.FactorizationMachineModel.VInitScheme;
+import hivemall.optimizer.EtaEstimator;
import hivemall.utils.lang.Primitives;
import javax.annotation.Nonnull;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/fm/FactorizationMachineModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/fm/FactorizationMachineModel.java b/core/src/main/java/hivemall/fm/FactorizationMachineModel.java
index 396328a..4b6ece6 100644
--- a/core/src/main/java/hivemall/fm/FactorizationMachineModel.java
+++ b/core/src/main/java/hivemall/fm/FactorizationMachineModel.java
@@ -18,7 +18,7 @@
*/
package hivemall.fm;
-import hivemall.common.EtaEstimator;
+import hivemall.optimizer.EtaEstimator;
import hivemall.utils.lang.NumberUtils;
import hivemall.utils.math.MathUtils;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java b/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java
index 2388689..7739c52 100644
--- a/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java
+++ b/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java
@@ -20,10 +20,10 @@ package hivemall.fm;
import hivemall.UDTFWithOptions;
import hivemall.common.ConversionState;
-import hivemall.common.EtaEstimator;
-import hivemall.common.LossFunctions;
-import hivemall.common.LossFunctions.LossFunction;
-import hivemall.common.LossFunctions.LossType;
+import hivemall.optimizer.EtaEstimator;
+import hivemall.optimizer.LossFunctions;
+import hivemall.optimizer.LossFunctions.LossFunction;
+import hivemall.optimizer.LossFunctions.LossType;
import hivemall.fm.FMStringFeatureMapModel.Entry;
import hivemall.utils.collections.IMapIterator;
import hivemall.utils.hadoop.HiveUtils;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/fm/FieldAwareFactorizationMachineModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/fm/FieldAwareFactorizationMachineModel.java b/core/src/main/java/hivemall/fm/FieldAwareFactorizationMachineModel.java
index 7e3cc50..fde7701 100644
--- a/core/src/main/java/hivemall/fm/FieldAwareFactorizationMachineModel.java
+++ b/core/src/main/java/hivemall/fm/FieldAwareFactorizationMachineModel.java
@@ -21,6 +21,7 @@ package hivemall.fm;
import hivemall.fm.FMHyperParameters.FFMHyperParameters;
import hivemall.utils.collections.DoubleArray3D;
import hivemall.utils.collections.IntArrayList;
+import hivemall.optimizer.EtaEstimator;
import hivemall.utils.lang.NumberUtils;
import java.util.Arrays;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/mf/BPRMatrixFactorizationUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/mf/BPRMatrixFactorizationUDTF.java b/core/src/main/java/hivemall/mf/BPRMatrixFactorizationUDTF.java
index d859f29..87d2654 100644
--- a/core/src/main/java/hivemall/mf/BPRMatrixFactorizationUDTF.java
+++ b/core/src/main/java/hivemall/mf/BPRMatrixFactorizationUDTF.java
@@ -20,7 +20,7 @@ package hivemall.mf;
import hivemall.UDTFWithOptions;
import hivemall.common.ConversionState;
-import hivemall.common.EtaEstimator;
+import hivemall.optimizer.EtaEstimator;
import hivemall.mf.FactorizedModel.RankInitScheme;
import hivemall.utils.hadoop.HiveUtils;
import hivemall.utils.io.FileUtils;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/mf/MatrixFactorizationSGDUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/mf/MatrixFactorizationSGDUDTF.java b/core/src/main/java/hivemall/mf/MatrixFactorizationSGDUDTF.java
index 317da85..ab79ce2 100644
--- a/core/src/main/java/hivemall/mf/MatrixFactorizationSGDUDTF.java
+++ b/core/src/main/java/hivemall/mf/MatrixFactorizationSGDUDTF.java
@@ -18,7 +18,7 @@
*/
package hivemall.mf;
-import hivemall.common.EtaEstimator;
+import hivemall.optimizer.EtaEstimator;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/model/DenseModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/model/DenseModel.java b/core/src/main/java/hivemall/model/DenseModel.java
index ee57574..6956875 100644
--- a/core/src/main/java/hivemall/model/DenseModel.java
+++ b/core/src/main/java/hivemall/model/DenseModel.java
@@ -18,21 +18,18 @@
*/
package hivemall.model;
-import hivemall.model.WeightValue.WeightValueParamsF1;
-import hivemall.model.WeightValue.WeightValueParamsF2;
-import hivemall.model.WeightValue.WeightValueWithCovar;
-import hivemall.utils.collections.IMapIterator;
-import hivemall.utils.hadoop.HiveUtils;
-import hivemall.utils.lang.Copyable;
-import hivemall.utils.math.MathUtils;
-
import java.util.Arrays;
-
import javax.annotation.Nonnull;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import hivemall.model.WeightValue.WeightValueWithCovar;
+import hivemall.utils.collections.IMapIterator;
+import hivemall.utils.hadoop.HiveUtils;
+import hivemall.utils.lang.Copyable;
+import hivemall.utils.math.MathUtils;
+
public final class DenseModel extends AbstractPredictionModel {
private static final Log logger = LogFactory.getLog(DenseModel.class);
@@ -40,13 +37,6 @@ public final class DenseModel extends AbstractPredictionModel {
private float[] weights;
private float[] covars;
- // optional values for adagrad
- private float[] sum_of_squared_gradients;
- // optional value for adadelta
- private float[] sum_of_squared_delta_x;
- // optional value for adagrad+rda
- private float[] sum_of_gradients;
-
// optional value for MIX
private short[] clocks;
private byte[] deltaUpdates;
@@ -67,9 +57,6 @@ public final class DenseModel extends AbstractPredictionModel {
} else {
this.covars = null;
}
- this.sum_of_squared_gradients = null;
- this.sum_of_squared_delta_x = null;
- this.sum_of_gradients = null;
this.clocks = null;
this.deltaUpdates = null;
}
@@ -85,20 +72,6 @@ public final class DenseModel extends AbstractPredictionModel {
}
@Override
- public void configureParams(boolean sum_of_squared_gradients, boolean sum_of_squared_delta_x,
- boolean sum_of_gradients) {
- if (sum_of_squared_gradients) {
- this.sum_of_squared_gradients = new float[size];
- }
- if (sum_of_squared_delta_x) {
- this.sum_of_squared_delta_x = new float[size];
- }
- if (sum_of_gradients) {
- this.sum_of_gradients = new float[size];
- }
- }
-
- @Override
public void configureClock() {
if (clocks == null) {
this.clocks = new short[size];
@@ -129,16 +102,7 @@ public final class DenseModel extends AbstractPredictionModel {
this.covars = Arrays.copyOf(covars, newSize);
Arrays.fill(covars, oldSize, newSize, 1.f);
}
- if (sum_of_squared_gradients != null) {
- this.sum_of_squared_gradients = Arrays.copyOf(sum_of_squared_gradients, newSize);
- }
- if (sum_of_squared_delta_x != null) {
- this.sum_of_squared_delta_x = Arrays.copyOf(sum_of_squared_delta_x, newSize);
- }
- if (sum_of_gradients != null) {
- this.sum_of_gradients = Arrays.copyOf(sum_of_gradients, newSize);
- }
- if (clocks != null) {
+ if(clocks != null) {
this.clocks = Arrays.copyOf(clocks, newSize);
this.deltaUpdates = Arrays.copyOf(deltaUpdates, newSize);
}
@@ -152,17 +116,7 @@ public final class DenseModel extends AbstractPredictionModel {
if (i >= size) {
return null;
}
- if (sum_of_squared_gradients != null) {
- if (sum_of_squared_delta_x != null) {
- return (T) new WeightValueParamsF2(weights[i], sum_of_squared_gradients[i],
- sum_of_squared_delta_x[i]);
- } else if (sum_of_gradients != null) {
- return (T) new WeightValueParamsF2(weights[i], sum_of_squared_gradients[i],
- sum_of_gradients[i]);
- } else {
- return (T) new WeightValueParamsF1(weights[i], sum_of_squared_gradients[i]);
- }
- } else if (covars != null) {
+ if(covars != null) {
return (T) new WeightValueWithCovar(weights[i], covars[i]);
} else {
return (T) new WeightValue(weights[i]);
@@ -181,15 +135,6 @@ public final class DenseModel extends AbstractPredictionModel {
covar = value.getCovariance();
covars[i] = covar;
}
- if (sum_of_squared_gradients != null) {
- sum_of_squared_gradients[i] = value.getSumOfSquaredGradients();
- }
- if (sum_of_squared_delta_x != null) {
- sum_of_squared_delta_x[i] = value.getSumOfSquaredDeltaX();
- }
- if (sum_of_gradients != null) {
- sum_of_gradients[i] = value.getSumOfGradients();
- }
short clock = 0;
int delta = 0;
if (clocks != null && value.isTouched()) {
@@ -213,15 +158,6 @@ public final class DenseModel extends AbstractPredictionModel {
if (covars != null) {
covars[i] = 1.f;
}
- if (sum_of_squared_gradients != null) {
- sum_of_squared_gradients[i] = 0.f;
- }
- if (sum_of_squared_delta_x != null) {
- sum_of_squared_delta_x[i] = 0.f;
- }
- if (sum_of_gradients != null) {
- sum_of_gradients[i] = 0.f;
- }
// avoid clock/delta
}
@@ -235,6 +171,13 @@ public final class DenseModel extends AbstractPredictionModel {
}
@Override
+ public void setWeight(Object feature, float value) {
+ int i = HiveUtils.parseInt(feature);
+ ensureCapacity(i);
+ weights[i] = value;
+ }
+
+ @Override
public float getCovariance(Object feature) {
int i = HiveUtils.parseInt(feature);
if (i >= size) {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/model/IWeightValue.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/model/IWeightValue.java b/core/src/main/java/hivemall/model/IWeightValue.java
index 988e4a1..259628f 100644
--- a/core/src/main/java/hivemall/model/IWeightValue.java
+++ b/core/src/main/java/hivemall/model/IWeightValue.java
@@ -25,7 +25,7 @@ import javax.annotation.Nonnegative;
public interface IWeightValue extends Copyable<IWeightValue> {
public enum WeightValueType {
- NoParams, ParamsF1, ParamsF2, ParamsCovar;
+ NoParams, ParamsF1, ParamsF2, ParamsF3, ParamsCovar;
}
WeightValueType getType();
@@ -44,10 +44,24 @@ public interface IWeightValue extends Copyable<IWeightValue> {
float getSumOfSquaredGradients();
+ void setSumOfSquaredGradients(float value);
+
float getSumOfSquaredDeltaX();
+ void setSumOfSquaredDeltaX(float value);
+
float getSumOfGradients();
+ void setSumOfGradients(float value);
+
+ float getM();
+
+ void setM(float value);
+
+ float getV();
+
+ void setV(float value);
+
/**
* @return whether touched in training or not
*/
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/model/PredictionModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/model/PredictionModel.java b/core/src/main/java/hivemall/model/PredictionModel.java
index a8efee0..8d8dd2b 100644
--- a/core/src/main/java/hivemall/model/PredictionModel.java
+++ b/core/src/main/java/hivemall/model/PredictionModel.java
@@ -34,9 +34,6 @@ public interface PredictionModel extends MixedModel {
boolean hasCovariance();
- void configureParams(boolean sum_of_squared_gradients, boolean sum_of_squared_delta_x,
- boolean sum_of_gradients);
-
void configureClock();
boolean hasClock();
@@ -56,6 +53,8 @@ public interface PredictionModel extends MixedModel {
float getWeight(@Nonnull Object feature);
+ void setWeight(@Nonnull Object feature, float value);
+
float getCovariance(@Nonnull Object feature);
<K, V extends IWeightValue> IMapIterator<K, V> entries();
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/model/SpaceEfficientDenseModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/model/SpaceEfficientDenseModel.java b/core/src/main/java/hivemall/model/SpaceEfficientDenseModel.java
index b3cd3ff..8b668e7 100644
--- a/core/src/main/java/hivemall/model/SpaceEfficientDenseModel.java
+++ b/core/src/main/java/hivemall/model/SpaceEfficientDenseModel.java
@@ -18,8 +18,6 @@
*/
package hivemall.model;
-import hivemall.model.WeightValue.WeightValueParamsF1;
-import hivemall.model.WeightValue.WeightValueParamsF2;
import hivemall.model.WeightValue.WeightValueWithCovar;
import hivemall.utils.collections.IMapIterator;
import hivemall.utils.hadoop.HiveUtils;
@@ -28,7 +26,6 @@ import hivemall.utils.lang.HalfFloat;
import hivemall.utils.math.MathUtils;
import java.util.Arrays;
-
import javax.annotation.Nonnull;
import org.apache.commons.logging.Log;
@@ -41,13 +38,6 @@ public final class SpaceEfficientDenseModel extends AbstractPredictionModel {
private short[] weights;
private short[] covars;
- // optional value for adagrad
- private float[] sum_of_squared_gradients;
- // optional value for adadelta
- private float[] sum_of_squared_delta_x;
- // optional value for adagrad+rda
- private float[] sum_of_gradients;
-
// optional value for MIX
private short[] clocks;
private byte[] deltaUpdates;
@@ -68,9 +58,6 @@ public final class SpaceEfficientDenseModel extends AbstractPredictionModel {
} else {
this.covars = null;
}
- this.sum_of_squared_gradients = null;
- this.sum_of_squared_delta_x = null;
- this.sum_of_gradients = null;
this.clocks = null;
this.deltaUpdates = null;
}
@@ -86,20 +73,6 @@ public final class SpaceEfficientDenseModel extends AbstractPredictionModel {
}
@Override
- public void configureParams(boolean sum_of_squared_gradients, boolean sum_of_squared_delta_x,
- boolean sum_of_gradients) {
- if (sum_of_squared_gradients) {
- this.sum_of_squared_gradients = new float[size];
- }
- if (sum_of_squared_delta_x) {
- this.sum_of_squared_delta_x = new float[size];
- }
- if (sum_of_gradients) {
- this.sum_of_gradients = new float[size];
- }
- }
-
- @Override
public void configureClock() {
if (clocks == null) {
this.clocks = new short[size];
@@ -126,8 +99,11 @@ public final class SpaceEfficientDenseModel extends AbstractPredictionModel {
return HalfFloat.halfFloatToFloat(covars[i]);
}
- private void setWeight(final int i, final float v) {
- HalfFloat.checkRange(v);
+ private void _setWeight(final int i, final float v) {
+ if(Math.abs(v) >= HalfFloat.MAX_FLOAT) {
+ throw new IllegalArgumentException("Acceptable maximum weight is "
+ + HalfFloat.MAX_FLOAT + ": " + v);
+ }
weights[i] = HalfFloat.floatToHalfFloat(v);
}
@@ -149,16 +125,7 @@ public final class SpaceEfficientDenseModel extends AbstractPredictionModel {
this.covars = Arrays.copyOf(covars, newSize);
Arrays.fill(covars, oldSize, newSize, HalfFloat.ONE);
}
- if (sum_of_squared_gradients != null) {
- this.sum_of_squared_gradients = Arrays.copyOf(sum_of_squared_gradients, newSize);
- }
- if (sum_of_squared_delta_x != null) {
- this.sum_of_squared_delta_x = Arrays.copyOf(sum_of_squared_delta_x, newSize);
- }
- if (sum_of_gradients != null) {
- this.sum_of_gradients = Arrays.copyOf(sum_of_gradients, newSize);
- }
- if (clocks != null) {
+ if(clocks != null) {
this.clocks = Arrays.copyOf(clocks, newSize);
this.deltaUpdates = Arrays.copyOf(deltaUpdates, newSize);
}
@@ -172,17 +139,8 @@ public final class SpaceEfficientDenseModel extends AbstractPredictionModel {
if (i >= size) {
return null;
}
- if (sum_of_squared_gradients != null) {
- if (sum_of_squared_delta_x != null) {
- return (T) new WeightValueParamsF2(getWeight(i), sum_of_squared_gradients[i],
- sum_of_squared_delta_x[i]);
- } else if (sum_of_gradients != null) {
- return (T) new WeightValueParamsF2(getWeight(i), sum_of_squared_gradients[i],
- sum_of_gradients[i]);
- } else {
- return (T) new WeightValueParamsF1(getWeight(i), sum_of_squared_gradients[i]);
- }
- } else if (covars != null) {
+
+ if(covars != null) {
return (T) new WeightValueWithCovar(getWeight(i), getCovar(i));
} else {
return (T) new WeightValue(getWeight(i));
@@ -194,22 +152,13 @@ public final class SpaceEfficientDenseModel extends AbstractPredictionModel {
int i = HiveUtils.parseInt(feature);
ensureCapacity(i);
float weight = value.get();
- setWeight(i, weight);
+ _setWeight(i, weight);
float covar = 1.f;
boolean hasCovar = value.hasCovariance();
if (hasCovar) {
covar = value.getCovariance();
setCovar(i, covar);
}
- if (sum_of_squared_gradients != null) {
- sum_of_squared_gradients[i] = value.getSumOfSquaredGradients();
- }
- if (sum_of_squared_delta_x != null) {
- sum_of_squared_delta_x[i] = value.getSumOfSquaredDeltaX();
- }
- if (sum_of_gradients != null) {
- sum_of_gradients[i] = value.getSumOfGradients();
- }
short clock = 0;
int delta = 0;
if (clocks != null && value.isTouched()) {
@@ -229,19 +178,10 @@ public final class SpaceEfficientDenseModel extends AbstractPredictionModel {
if (i >= size) {
return;
}
- setWeight(i, 0.f);
- if (covars != null) {
+ _setWeight(i, 0.f);
+ if(covars != null) {
setCovar(i, 1.f);
}
- if (sum_of_squared_gradients != null) {
- sum_of_squared_gradients[i] = 0.f;
- }
- if (sum_of_squared_delta_x != null) {
- sum_of_squared_delta_x[i] = 0.f;
- }
- if (sum_of_gradients != null) {
- sum_of_gradients[i] = 0.f;
- }
// avoid clock/delta
}
@@ -255,6 +195,13 @@ public final class SpaceEfficientDenseModel extends AbstractPredictionModel {
}
@Override
+ public void setWeight(Object feature, float value) {
+ int i = HiveUtils.parseInt(feature);
+ ensureCapacity(i);
+ _setWeight(i, value);
+ }
+
+ @Override
public float getCovariance(Object feature) {
int i = HiveUtils.parseInt(feature);
if (i >= size) {
@@ -267,7 +214,7 @@ public final class SpaceEfficientDenseModel extends AbstractPredictionModel {
protected void _set(Object feature, float weight, short clock) {
int i = ((Integer) feature).intValue();
ensureCapacity(i);
- setWeight(i, weight);
+ _setWeight(i, weight);
clocks[i] = clock;
deltaUpdates[i] = 0;
}
@@ -276,7 +223,7 @@ public final class SpaceEfficientDenseModel extends AbstractPredictionModel {
protected void _set(Object feature, float weight, float covar, short clock) {
int i = ((Integer) feature).intValue();
ensureCapacity(i);
- setWeight(i, weight);
+ _setWeight(i, weight);
setCovar(i, covar);
clocks[i] = clock;
deltaUpdates[i] = 0;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/model/SparseModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/model/SparseModel.java b/core/src/main/java/hivemall/model/SparseModel.java
index aaab869..bab982f 100644
--- a/core/src/main/java/hivemall/model/SparseModel.java
+++ b/core/src/main/java/hivemall/model/SparseModel.java
@@ -36,6 +36,10 @@ public final class SparseModel extends AbstractPredictionModel {
private final boolean hasCovar;
private boolean clockEnabled;
+ public SparseModel(int size) {
+ this(size, false);
+ }
+
public SparseModel(int size, boolean hasCovar) {
super();
this.weights = new OpenHashMap<Object, IWeightValue>(size);
@@ -54,10 +58,6 @@ public final class SparseModel extends AbstractPredictionModel {
}
@Override
- public void configureParams(boolean sum_of_squared_gradients, boolean sum_of_squared_delta_x,
- boolean sum_of_gradients) {}
-
- @Override
public void configureClock() {
this.clockEnabled = true;
}
@@ -131,6 +131,18 @@ public final class SparseModel extends AbstractPredictionModel {
}
@Override
+ public void setWeight(Object feature, float value) {
+ if(weights.containsKey(feature)) {
+ IWeightValue weight = weights.get(feature);
+ weight.set(value);
+ } else {
+ IWeightValue weight = new WeightValue(value);
+ weight.set(value);
+ weights.put(feature, weight);
+ }
+ }
+
+ @Override
public float getCovariance(final Object feature) {
IWeightValue v = weights.get(feature);
return v == null ? 1.f : v.getCovariance();
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/model/SynchronizedModelWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/model/SynchronizedModelWrapper.java b/core/src/main/java/hivemall/model/SynchronizedModelWrapper.java
index 99ee69c..87e89b6 100644
--- a/core/src/main/java/hivemall/model/SynchronizedModelWrapper.java
+++ b/core/src/main/java/hivemall/model/SynchronizedModelWrapper.java
@@ -63,12 +63,6 @@ public final class SynchronizedModelWrapper implements PredictionModel {
}
@Override
- public void configureParams(boolean sum_of_squared_gradients, boolean sum_of_squared_delta_x,
- boolean sum_of_gradients) {
- model.configureParams(sum_of_squared_gradients, sum_of_squared_delta_x, sum_of_gradients);
- }
-
- @Override
public void configureClock() {
model.configureClock();
}
@@ -157,6 +151,16 @@ public final class SynchronizedModelWrapper implements PredictionModel {
}
@Override
+ public void setWeight(Object feature, float value) {
+ try {
+ lock.lock();
+ model.setWeight(feature, value);
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
public float getCovariance(Object feature) {
try {
lock.lock();
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/model/WeightValue.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/model/WeightValue.java b/core/src/main/java/hivemall/model/WeightValue.java
index e6d98c6..b329374 100644
--- a/core/src/main/java/hivemall/model/WeightValue.java
+++ b/core/src/main/java/hivemall/model/WeightValue.java
@@ -77,15 +77,50 @@ public class WeightValue implements IWeightValue {
}
@Override
+ public void setSumOfSquaredGradients(float value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public float getSumOfSquaredDeltaX() {
return 0.f;
}
@Override
+ public void setSumOfSquaredDeltaX(float value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public float getSumOfGradients() {
return 0.f;
}
+ @Override
+ public void setSumOfGradients(float value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public float getM() {
+ return 0.f;
+ }
+
+ @Override
+ public void setM(float value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public float getV() {
+ return 0.f;
+ }
+
+ @Override
+ public void setV(float value) {
+ throw new UnsupportedOperationException();
+ }
+
/**
* @return whether touched in training or not
*/
@@ -137,7 +172,7 @@ public class WeightValue implements IWeightValue {
}
public static final class WeightValueParamsF1 extends WeightValue {
- private final float f1;
+ private float f1;
public WeightValueParamsF1(float weight, float f1) {
super(weight);
@@ -162,14 +197,19 @@ public class WeightValue implements IWeightValue {
return f1;
}
+ @Override
+ public void setSumOfSquaredGradients(float value) {
+ this.f1 = value;
+ }
+
}
/**
* WeightValue with Sum of Squared Gradients
*/
public static final class WeightValueParamsF2 extends WeightValue {
- private final float f1;
- private final float f2;
+ private float f1;
+ private float f2;
public WeightValueParamsF2(float weight, float f1, float f2) {
super(weight);
@@ -198,15 +238,131 @@ public class WeightValue implements IWeightValue {
}
@Override
+ public void setSumOfSquaredGradients(float value) {
+ this.f1 = value;
+ }
+
+ @Override
public final float getSumOfSquaredDeltaX() {
return f2;
}
@Override
+ public void setSumOfSquaredDeltaX(float value) {
+ this.f2 = value;
+ }
+
+ @Override
public float getSumOfGradients() {
return f2;
}
+ @Override
+ public void setSumOfGradients(float value) {
+ this.f2 = value;
+ }
+
+ @Override
+ public float getM() {
+ return f1;
+ }
+
+ @Override
+ public void setM(float value) {
+ this.f1 = value;
+ }
+
+ @Override
+ public float getV() {
+ return f2;
+ }
+
+ @Override
+ public void setV(float value) {
+ this.f2 = value;
+ }
+
+ }
+
+ public static final class WeightValueParamsF3 extends WeightValue {
+ private float f1;
+ private float f2;
+ private float f3;
+
+ public WeightValueParamsF3(float weight, float f1, float f2, float f3) {
+ super(weight);
+ this.f1 = f1;
+ this.f2 = f2;
+ this.f3 = f3;
+ }
+
+ @Override
+ public WeightValueType getType() {
+ return WeightValueType.ParamsF3;
+ }
+
+ @Override
+ public float getFloatParams(@Nonnegative final int i) {
+ if(i == 1) {
+ return f1;
+ } else if(i == 2) {
+ return f2;
+ } else if (i == 3) {
+ return f3;
+ }
+ throw new IllegalArgumentException("getFloatParams(" + i + ") should not be called");
+ }
+
+ @Override
+ public final float getSumOfSquaredGradients() {
+ return f1;
+ }
+
+ @Override
+ public void setSumOfSquaredGradients(float value) {
+ this.f1 = value;
+ }
+
+ @Override
+ public final float getSumOfSquaredDeltaX() {
+ return f2;
+ }
+
+ @Override
+ public void setSumOfSquaredDeltaX(float value) {
+ this.f2 = value;
+ }
+
+ @Override
+ public float getSumOfGradients() {
+ return f3;
+ }
+
+ @Override
+ public void setSumOfGradients(float value) {
+ this.f3 = value;
+ }
+
+ @Override
+ public float getM() {
+ return f1;
+ }
+
+ @Override
+ public void setM(float value) {
+ this.f1 = value;
+ }
+
+ @Override
+ public float getV() {
+ return f2;
+ }
+
+ @Override
+ public void setV(float value) {
+ this.f2 = value;
+ }
+
}
public static final class WeightValueWithCovar extends WeightValue {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/model/WeightValueWithClock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/model/WeightValueWithClock.java b/core/src/main/java/hivemall/model/WeightValueWithClock.java
index 249650a..9b31361 100644
--- a/core/src/main/java/hivemall/model/WeightValueWithClock.java
+++ b/core/src/main/java/hivemall/model/WeightValueWithClock.java
@@ -79,15 +79,50 @@ public class WeightValueWithClock implements IWeightValue {
}
@Override
+ public void setSumOfSquaredGradients(float value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public float getSumOfSquaredDeltaX() {
return 0.f;
}
@Override
+ public void setSumOfSquaredDeltaX(float value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public float getSumOfGradients() {
return 0.f;
}
+ @Override
+ public void setSumOfGradients(float value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public float getM() {
+ return 0.f;
+ }
+
+ @Override
+ public void setM(float value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public float getV() {
+ return 0.f;
+ }
+
+ @Override
+ public void setV(float value) {
+ throw new UnsupportedOperationException();
+ }
+
/**
* @return whether touched in training or not
*/
@@ -144,7 +179,7 @@ public class WeightValueWithClock implements IWeightValue {
* WeightValue with Sum of Squared Gradients
*/
public static final class WeightValueParamsF1Clock extends WeightValueWithClock {
- private final float f1;
+ private float f1;
public WeightValueParamsF1Clock(float value, float f1) {
super(value);
@@ -174,11 +209,16 @@ public class WeightValueWithClock implements IWeightValue {
return f1;
}
+ @Override
+ public void setSumOfSquaredGradients(float value) {
+ this.f1 = value;
+ }
+
}
public static final class WeightValueParamsF2Clock extends WeightValueWithClock {
- private final float f1;
- private final float f2;
+ private float f1;
+ private float f2;
public WeightValueParamsF2Clock(float value, float f1, float f2) {
super(value);
@@ -213,15 +253,136 @@ public class WeightValueWithClock implements IWeightValue {
}
@Override
+ public void setSumOfSquaredGradients(float value) {
+ this.f1 = value;
+ }
+
+ @Override
+ public float getSumOfSquaredDeltaX() {
+ return f2;
+ }
+
+ @Override
+ public void setSumOfSquaredDeltaX(float value) {
+ this.f2 = value;
+ }
+
+ @Override
+ public float getSumOfGradients() {
+ return f2;
+ }
+
+ @Override
+ public void setSumOfGradients(float value) {
+ this.f2 = value;
+ }
+ @Override
+ public float getM() {
+ return f1;
+ }
+
+ @Override
+ public void setM(float value) {
+ this.f1 = value;
+ }
+
+ @Override
+ public float getV() {
+ return f2;
+ }
+
+ @Override
+ public void setV(float value) {
+ this.f2 = value;
+ }
+
+ }
+
+ public static final class WeightValueParamsF3Clock extends WeightValueWithClock {
+ private float f1;
+ private float f2;
+ private float f3;
+
+ public WeightValueParamsF3Clock(float value, float f1, float f2, float f3) {
+ super(value);
+ this.f1 = f1;
+ this.f2 = f2;
+ this.f3 = f3;
+ }
+
+ public WeightValueParamsF3Clock(IWeightValue src) {
+ super(src);
+ this.f1 = src.getFloatParams(1);
+ this.f2 = src.getFloatParams(2);
+ this.f3 = src.getFloatParams(3);
+ }
+
+ @Override
+ public WeightValueType getType() {
+ return WeightValueType.ParamsF3;
+ }
+
+ @Override
+ public float getFloatParams(@Nonnegative final int i) {
+ if(i == 1) {
+ return f1;
+ } else if(i == 2) {
+ return f2;
+ } else if(i == 3) {
+ return f3;
+ }
+ throw new IllegalArgumentException("getFloatParams(" + i + ") should not be called");
+ }
+
+ @Override
+ public float getSumOfSquaredGradients() {
+ return f1;
+ }
+
+ @Override
+ public void setSumOfSquaredGradients(float value) {
+ this.f1 = value;
+ }
+
+ @Override
public float getSumOfSquaredDeltaX() {
return f2;
}
@Override
+ public void setSumOfSquaredDeltaX(float value) {
+ this.f2 = value;
+ }
+
+ @Override
public float getSumOfGradients() {
+ return f3;
+ }
+
+ @Override
+ public void setSumOfGradients(float value) {
+ this.f3 = value;
+ }
+ @Override
+ public float getM() {
+ return f1;
+ }
+
+ @Override
+ public void setM(float value) {
+ this.f1 = value;
+ }
+
+ @Override
+ public float getV() {
return f2;
}
+ @Override
+ public void setV(float value) {
+ this.f2 = value;
+ }
+
}
public static final class WeightValueWithCovarClock extends WeightValueWithClock {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/optimizer/DenseOptimizerFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/optimizer/DenseOptimizerFactory.java b/core/src/main/java/hivemall/optimizer/DenseOptimizerFactory.java
new file mode 100644
index 0000000..e2c5a10
--- /dev/null
+++ b/core/src/main/java/hivemall/optimizer/DenseOptimizerFactory.java
@@ -0,0 +1,215 @@
+/*
+ * Hivemall: Hive scalable Machine Learning Library
+ *
+ * Copyright (C) 2015 Makoto YUI
+ * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hivemall.optimizer;
+
+import javax.annotation.Nonnull;
+import javax.annotation.concurrent.NotThreadSafe;
+import java.util.Arrays;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import hivemall.optimizer.Optimizer.OptimizerBase;
+import hivemall.model.IWeightValue;
+import hivemall.model.WeightValue;
+import hivemall.utils.hadoop.HiveUtils;
+import hivemall.utils.math.MathUtils;
+
+public final class DenseOptimizerFactory {
+ private static final Log logger = LogFactory.getLog(DenseOptimizerFactory.class);
+
+ @Nonnull
+ public static Optimizer create(int ndims, @Nonnull Map<String, String> options) {
+ final String optimizerName = options.get("optimizer");
+ if(optimizerName != null) {
+ OptimizerBase optimizerImpl;
+ if(optimizerName.toLowerCase().equals("sgd")) {
+ optimizerImpl = new Optimizer.SGD(options);
+ } else if(optimizerName.toLowerCase().equals("adadelta")) {
+ optimizerImpl = new AdaDelta(ndims, options);
+ } else if(optimizerName.toLowerCase().equals("adagrad")) {
+ optimizerImpl = new AdaGrad(ndims, options);
+ } else if(optimizerName.toLowerCase().equals("adam")) {
+ optimizerImpl = new Adam(ndims, options);
+ } else {
+ throw new IllegalArgumentException("Unsupported optimizer name: " + optimizerName);
+ }
+
+ logger.info("set " + optimizerImpl.getClass().getSimpleName()
+ + " as an optimizer: " + options);
+
+ // If a regularization type is "RDA", wrap the optimizer with `Optimizer#RDA`.
+ if(options.get("regularization") != null
+ && options.get("regularization").toLowerCase().equals("rda")) {
+ optimizerImpl = new RDA(ndims, optimizerImpl, options);
+ }
+
+ return optimizerImpl;
+ }
+ throw new IllegalArgumentException("`optimizer` not defined");
+ }
+
+ @NotThreadSafe
+ static final class AdaDelta extends Optimizer.AdaDelta {
+
+ private final IWeightValue weightValueReused;
+
+ private float[] sum_of_squared_gradients;
+ private float[] sum_of_squared_delta_x;
+
+ public AdaDelta(int ndims, Map<String, String> options) {
+ super(options);
+ this.weightValueReused = new WeightValue.WeightValueParamsF2(0.f, 0.f, 0.f);
+ this.sum_of_squared_gradients = new float[ndims];
+ this.sum_of_squared_delta_x = new float[ndims];
+ }
+
+ @Override
+ public float computeUpdatedValue(@Nonnull Object feature, float weight, float gradient) {
+ int i = HiveUtils.parseInt(feature);
+ ensureCapacity(i);
+ weightValueReused.set(weight);
+ weightValueReused.setSumOfSquaredGradients(sum_of_squared_gradients[i]);
+ weightValueReused.setSumOfSquaredDeltaX(sum_of_squared_delta_x[i]);
+ computeUpdateValue(weightValueReused, gradient);
+ sum_of_squared_gradients[i] = weightValueReused.getSumOfSquaredGradients();
+ sum_of_squared_delta_x[i] = weightValueReused.getSumOfSquaredDeltaX();
+ return weightValueReused.get();
+ }
+
+ private void ensureCapacity(final int index) {
+ if(index >= sum_of_squared_gradients.length) {
+ int bits = MathUtils.bitsRequired(index);
+ int newSize = (1 << bits) + 1;
+ this.sum_of_squared_gradients = Arrays.copyOf(sum_of_squared_gradients, newSize);
+ this.sum_of_squared_delta_x = Arrays.copyOf(sum_of_squared_delta_x, newSize);
+ }
+ }
+
+ }
+
+ @NotThreadSafe
+ static final class AdaGrad extends Optimizer.AdaGrad {
+
+ private final IWeightValue weightValueReused;
+
+ private float[] sum_of_squared_gradients;
+
+ public AdaGrad(int ndims, Map<String, String> options) {
+ super(options);
+ this.weightValueReused = new WeightValue.WeightValueParamsF1(0.f, 0.f);
+ this.sum_of_squared_gradients = new float[ndims];
+ }
+
+ @Override
+ public float computeUpdatedValue(@Nonnull Object feature, float weight, float gradient) {
+ int i = HiveUtils.parseInt(feature);
+ ensureCapacity(i);
+ weightValueReused.set(weight);
+ weightValueReused.setSumOfSquaredGradients(sum_of_squared_gradients[i]);
+ computeUpdateValue(weightValueReused, gradient);
+ sum_of_squared_gradients[i] = weightValueReused.getSumOfSquaredGradients();
+ return weightValueReused.get();
+ }
+
+ private void ensureCapacity(final int index) {
+ if(index >= sum_of_squared_gradients.length) {
+ int bits = MathUtils.bitsRequired(index);
+ int newSize = (1 << bits) + 1;
+ this.sum_of_squared_gradients = Arrays.copyOf(sum_of_squared_gradients, newSize);
+ }
+ }
+
+ }
+
+ @NotThreadSafe
+ static final class Adam extends Optimizer.Adam {
+
+ private final IWeightValue weightValueReused;
+
+ private float[] val_m;
+ private float[] val_v;
+
+ public Adam(int ndims, Map<String, String> options) {
+ super(options);
+ this.weightValueReused = new WeightValue.WeightValueParamsF2(0.f, 0.f, 0.f);
+ this.val_m = new float[ndims];
+ this.val_v = new float[ndims];
+ }
+
+ @Override
+ public float computeUpdatedValue(@Nonnull Object feature, float weight, float gradient) {
+ int i = HiveUtils.parseInt(feature);
+ ensureCapacity(i);
+ weightValueReused.set(weight);
+ weightValueReused.setM(val_m[i]);
+ weightValueReused.setV(val_v[i]);
+ computeUpdateValue(weightValueReused, gradient);
+ val_m[i] = weightValueReused.getM();
+ val_v[i] = weightValueReused.getV();
+ return weightValueReused.get();
+ }
+
+ private void ensureCapacity(final int index) {
+ if(index >= val_m.length) {
+ int bits = MathUtils.bitsRequired(index);
+ int newSize = (1 << bits) + 1;
+ this.val_m = Arrays.copyOf(val_m, newSize);
+ this.val_v = Arrays.copyOf(val_v, newSize);
+ }
+ }
+
+ }
+
+ @NotThreadSafe
+ static final class RDA extends Optimizer.RDA {
+
+ private final IWeightValue weightValueReused;
+
+ private float[] sum_of_gradients;
+
+ public RDA(int ndims, final OptimizerBase optimizerImpl, Map<String, String> options) {
+ super(optimizerImpl, options);
+ this.weightValueReused = new WeightValue.WeightValueParamsF3(0.f, 0.f, 0.f, 0.f);
+ this.sum_of_gradients = new float[ndims];
+ }
+
+ @Override
+ public float computeUpdatedValue(@Nonnull Object feature, float weight, float gradient) {
+ int i = HiveUtils.parseInt(feature);
+ ensureCapacity(i);
+ weightValueReused.set(weight);
+ weightValueReused.setSumOfGradients(sum_of_gradients[i]);
+ computeUpdateValue(weightValueReused, gradient);
+ sum_of_gradients[i] = weightValueReused.getSumOfGradients();
+ return weightValueReused.get();
+ }
+
+ private void ensureCapacity(final int index) {
+ if(index >= sum_of_gradients.length) {
+ int bits = MathUtils.bitsRequired(index);
+ int newSize = (1 << bits) + 1;
+ this.sum_of_gradients = Arrays.copyOf(sum_of_gradients, newSize);
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/f81948c5/core/src/main/java/hivemall/optimizer/EtaEstimator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/optimizer/EtaEstimator.java b/core/src/main/java/hivemall/optimizer/EtaEstimator.java
new file mode 100644
index 0000000..ac1d112
--- /dev/null
+++ b/core/src/main/java/hivemall/optimizer/EtaEstimator.java
@@ -0,0 +1,191 @@
+/*
+ * Hivemall: Hive scalable Machine Learning Library
+ *
+ * Copyright (C) 2015 Makoto YUI
+ * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hivemall.optimizer;
+
+import hivemall.utils.lang.NumberUtils;
+import hivemall.utils.lang.Primitives;
+
+import java.util.Map;
+import javax.annotation.Nonnegative;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+
+public abstract class EtaEstimator {
+
+ protected final float eta0;
+
+ public EtaEstimator(float eta0) {
+ this.eta0 = eta0;
+ }
+
+ public float eta0() {
+ return eta0;
+ }
+
+ public abstract float eta(long t);
+
+ public void update(@Nonnegative float multipler) {}
+
+ public static final class FixedEtaEstimator extends EtaEstimator {
+
+ public FixedEtaEstimator(float eta) {
+ super(eta);
+ }
+
+ @Override
+ public float eta(long t) {
+ return eta0;
+ }
+
+ }
+
+ public static final class SimpleEtaEstimator extends EtaEstimator {
+
+ private final float finalEta;
+ private final double total_steps;
+
+ public SimpleEtaEstimator(float eta0, long total_steps) {
+ super(eta0);
+ this.finalEta = (float) (eta0 / 2.d);
+ this.total_steps = total_steps;
+ }
+
+ @Override
+ public float eta(final long t) {
+ if (t > total_steps) {
+ return finalEta;
+ }
+ return (float) (eta0 / (1.d + (t / total_steps)));
+ }
+
+ }
+
+ public static final class InvscalingEtaEstimator extends EtaEstimator {
+
+ private final double power_t;
+
+ public InvscalingEtaEstimator(float eta0, double power_t) {
+ super(eta0);
+ this.power_t = power_t;
+ }
+
+ @Override
+ public float eta(final long t) {
+ return (float) (eta0 / Math.pow(t, power_t));
+ }
+
+ }
+
+ /**
+ * bold driver: Gemulla et al., Large-scale matrix factorization with distributed stochastic
+ * gradient descent, KDD 2011.
+ */
+ public static final class AdjustingEtaEstimator extends EtaEstimator {
+
+ private float eta;
+
+ public AdjustingEtaEstimator(float eta) {
+ super(eta);
+ this.eta = eta;
+ }
+
+ @Override
+ public float eta(long t) {
+ return eta;
+ }
+
+ @Override
+ public void update(@Nonnegative float multipler) {
+ float newEta = eta * multipler;
+ if (!NumberUtils.isFinite(newEta)) {
+ // avoid NaN or INFINITY
+ return;
+ }
+ this.eta = Math.min(eta0, newEta); // never be larger than eta0
+ }
+
+ }
+
+ @Nonnull
+ public static EtaEstimator get(@Nullable CommandLine cl) throws UDFArgumentException {
+ return get(cl, 0.1f);
+ }
+
+ @Nonnull
+ public static EtaEstimator get(@Nullable CommandLine cl, float defaultEta0)
+ throws UDFArgumentException {
+ if (cl == null) {
+ return new InvscalingEtaEstimator(defaultEta0, 0.1d);
+ }
+
+ if (cl.hasOption("boldDriver")) {
+ float eta = Primitives.parseFloat(cl.getOptionValue("eta"), 0.3f);
+ return new AdjustingEtaEstimator(eta);
+ }
+
+ String etaValue = cl.getOptionValue("eta");
+ if (etaValue != null) {
+ float eta = Float.parseFloat(etaValue);
+ return new FixedEtaEstimator(eta);
+ }
+
+ float eta0 = Primitives.parseFloat(cl.getOptionValue("eta0"), defaultEta0);
+ if (cl.hasOption("t")) {
+ long t = Long.parseLong(cl.getOptionValue("t"));
+ return new SimpleEtaEstimator(eta0, t);
+ }
+
+ double power_t = Primitives.parseDouble(cl.getOptionValue("power_t"), 0.1d);
+ return new InvscalingEtaEstimator(eta0, power_t);
+ }
+
+ @Nonnull
+ public static EtaEstimator get(@Nonnull final Map<String, String> options)
+ throws IllegalArgumentException {
+ final String etaName = options.get("eta");
+ if(etaName == null) {
+ return new FixedEtaEstimator(1.f);
+ }
+ float eta0 = 0.1f;
+ if(options.containsKey("eta0")) {
+ eta0 = Float.parseFloat(options.get("eta0"));
+ }
+ if(etaName.toLowerCase().equals("fixed")) {
+ return new FixedEtaEstimator(eta0);
+ } else if(etaName.toLowerCase().equals("simple")) {
+ long t = 10000;
+ if(options.containsKey("t")) {
+ t = Long.parseLong(options.get("t"));
+ }
+ return new SimpleEtaEstimator(eta0, t);
+ } else if(etaName.toLowerCase().equals("inverse")) {
+ double power_t = 0.1;
+ if(options.containsKey("power_t")) {
+ power_t = Double.parseDouble(options.get("power_t"));
+ }
+ return new InvscalingEtaEstimator(eta0, power_t);
+ } else {
+ throw new IllegalArgumentException("Unsupported ETA name: " + etaName);
+ }
+ }
+
+}