You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ch...@apache.org on 2018/01/09 19:14:40 UTC
[3/4] ignite git commit: IGNITE-7322: Return authorship of 7322 and
6783, step 2.
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingLoopData.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingLoopData.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingLoopData.java
new file mode 100644
index 0000000..0f3d974
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingLoopData.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn.trainers.distributed;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.functions.IgniteSupplier;
+import org.apache.ignite.ml.nn.MultilayerPerceptron;
+import org.apache.ignite.ml.nn.updaters.ParameterUpdateCalculator;
+import org.apache.ignite.ml.trainers.group.GroupTrainerCacheKey;
+
+/** Multilayer perceptron group update training loop data. */
+public class MLPGroupUpdateTrainingLoopData<P> implements Serializable {
+ /** */
+ private final ParameterUpdateCalculator<MultilayerPerceptron, P> updateCalculator;
+ /** */
+ private final int stepsCnt;
+ /** */
+ private final IgniteFunction<List<P>, P> updateReducer;
+ /** */
+ private final P previousUpdate;
+ /** */
+ private final IgniteSupplier<IgniteBiTuple<Matrix, Matrix>> batchSupplier;
+ /** */
+ private final IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
+ /** */
+ private final double tolerance;
+
+ /** */
+ private final GroupTrainerCacheKey<Void> key;
+ /** */
+ private final MultilayerPerceptron mlp;
+
+ /** Create multilayer perceptron group update training loop data. */
+ public MLPGroupUpdateTrainingLoopData(MultilayerPerceptron mlp,
+ ParameterUpdateCalculator<MultilayerPerceptron, P> updateCalculator, int stepsCnt,
+ IgniteFunction<List<P>, P> updateReducer, P previousUpdate,
+ GroupTrainerCacheKey<Void> key, IgniteSupplier<IgniteBiTuple<Matrix, Matrix>> batchSupplier,
+ IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss,
+ double tolerance) {
+ this.mlp = mlp;
+ this.updateCalculator = updateCalculator;
+ this.stepsCnt = stepsCnt;
+ this.updateReducer = updateReducer;
+ this.previousUpdate = previousUpdate;
+ this.key = key;
+ this.batchSupplier = batchSupplier;
+ this.loss = loss;
+ this.tolerance = tolerance;
+ }
+
+ /** Get perceptron. */
+ public MultilayerPerceptron mlp() {
+ return mlp;
+ }
+
+ /** Get update calculator. */
+ public ParameterUpdateCalculator<MultilayerPerceptron, P> updateCalculator() {
+ return updateCalculator;
+ }
+
+ /** Get steps count. */
+ public int stepsCnt() {
+ return stepsCnt;
+ }
+
+ /** Get update reducer. */
+ public IgniteFunction<List<P>, P> getUpdateReducer() {
+ return updateReducer;
+ }
+
+ /** Get previous update. */
+ public P previousUpdate() {
+ return previousUpdate;
+ }
+
+ /** Get group trainer cache key. */
+ public GroupTrainerCacheKey<Void> key() {
+ return key;
+ }
+
+ /** Get batch supplier. */
+ public IgniteSupplier<IgniteBiTuple<Matrix, Matrix>> batchSupplier() {
+ return batchSupplier;
+ }
+
+ /** Get loss function. */
+ public IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss() {
+ return loss;
+ }
+
+ /** Get tolerance. */
+ public double tolerance() {
+ return tolerance;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPMetaoptimizer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPMetaoptimizer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPMetaoptimizer.java
new file mode 100644
index 0000000..249136b
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPMetaoptimizer.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn.trainers.distributed;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.trainers.group.Metaoptimizer;
+
+/** Meta-optimizer for multilayer perceptron. */
+public class MLPMetaoptimizer<P> implements Metaoptimizer<MLPGroupUpdateTrainerLocalContext,
+ MLPGroupUpdateTrainingLoopData<P>, P, P, P, ArrayList<P>> {
+ /** */
+ private final IgniteFunction<List<P>, P> allUpdatesReducer;
+
+ /** Construct metaoptimizer. */
+ public MLPMetaoptimizer(IgniteFunction<List<P>, P> allUpdatesReducer) {
+ this.allUpdatesReducer = allUpdatesReducer;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteFunction<List<P>, P> initialReducer() {
+ return allUpdatesReducer;
+ }
+
+ /** {@inheritDoc} */
+ @Override public P locallyProcessInitData(P data, MLPGroupUpdateTrainerLocalContext locCtx) {
+ return data;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteFunction<P, ArrayList<P>> distributedPostprocessor() {
+ return p -> {
+ ArrayList<P> res = new ArrayList<>();
+ res.add(p);
+ return res;
+ };
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteFunction<List<ArrayList<P>>, ArrayList<P>> postProcessReducer() {
+ // Flatten.
+ return lists -> new ArrayList<>(lists.stream()
+ .flatMap(List::stream)
+ .collect(Collectors.toList()));
+ }
+
+ /** {@inheritDoc} */
+ @Override public P localProcessor(ArrayList<P> input, MLPGroupUpdateTrainerLocalContext locCtx) {
+ locCtx.incrementCurrentStep();
+
+ return allUpdatesReducer.apply(input);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean shouldContinue(P input, MLPGroupUpdateTrainerLocalContext locCtx) {
+ return input != null && locCtx.currentStep() < locCtx.globalStepsMaxCount();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/LocalBatchTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/LocalBatchTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/LocalBatchTrainer.java
index 64a1956..8579b82 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/LocalBatchTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/LocalBatchTrainer.java
@@ -28,19 +28,18 @@ import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.ml.math.functions.IgniteSupplier;
import org.apache.ignite.ml.math.util.MatrixUtil;
import org.apache.ignite.ml.nn.LocalBatchTrainerInput;
-import org.apache.ignite.ml.nn.updaters.ParameterUpdater;
-import org.apache.ignite.ml.nn.updaters.UpdaterParams;
+import org.apache.ignite.ml.nn.updaters.ParameterUpdateCalculator;
/**
* Batch trainer. This trainer is not distributed on the cluster, but input can theoretically read data from
* Ignite cache.
*/
-public class LocalBatchTrainer<M extends Model<Matrix, Matrix>, P extends UpdaterParams<? super M>>
+public class LocalBatchTrainer<M extends Model<Matrix, Matrix>, P>
implements Trainer<M, LocalBatchTrainerInput<M>> {
/**
* Supplier for updater function.
*/
- private final IgniteSupplier<ParameterUpdater<? super M, P>> updaterSupplier;
+ private final IgniteSupplier<ParameterUpdateCalculator<M, P>> updaterSupplier;
/**
* Error threshold.
@@ -71,7 +70,7 @@ public class LocalBatchTrainer<M extends Model<Matrix, Matrix>, P extends Update
* @param maxIterations Maximal iterations count.
*/
public LocalBatchTrainer(IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss,
- IgniteSupplier<ParameterUpdater<? super M, P>> updaterSupplier, double errorThreshold, int maxIterations) {
+ IgniteSupplier<ParameterUpdateCalculator<M, P>> updaterSupplier, double errorThreshold, int maxIterations) {
this.loss = loss;
this.updaterSupplier = updaterSupplier;
this.errorThreshold = errorThreshold;
@@ -84,19 +83,19 @@ public class LocalBatchTrainer<M extends Model<Matrix, Matrix>, P extends Update
M mdl = data.mdl();
double err;
- ParameterUpdater<? super M, P> updater = updaterSupplier.get();
+ ParameterUpdateCalculator<? super M, P> updater = updaterSupplier.get();
P updaterParams = updater.init(mdl, loss);
while (i < maxIterations) {
- IgniteBiTuple<Matrix, Matrix> batch = data.getBatch();
+ IgniteBiTuple<Matrix, Matrix> batch = data.batchSupplier().get();
Matrix input = batch.get1();
Matrix truth = batch.get2();
- updaterParams = updater.updateParams(mdl, updaterParams, i, input, truth);
+ updaterParams = updater.calculateNewUpdate(mdl, updaterParams, i, input, truth);
// Update mdl with updater parameters.
- mdl = updaterParams.update(mdl);
+ mdl = updater.update(mdl, updaterParams);
Matrix predicted = mdl.apply(input);
@@ -132,7 +131,7 @@ public class LocalBatchTrainer<M extends Model<Matrix, Matrix>, P extends Update
* @param updaterSupplier New updater supplier.
* @return new trainer with the same parameters as this trainer, but with new updater supplier.
*/
- public LocalBatchTrainer withUpdater(IgniteSupplier<ParameterUpdater<? super M, P>> updaterSupplier) {
+ public LocalBatchTrainer withUpdater(IgniteSupplier<ParameterUpdateCalculator<M, P>> updaterSupplier) {
return new LocalBatchTrainer<>(loss, updaterSupplier, errorThreshold, maxIterations);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/MLPLocalBatchTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/MLPLocalBatchTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/MLPLocalBatchTrainer.java
index 7065e2f..0c92395 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/MLPLocalBatchTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/MLPLocalBatchTrainer.java
@@ -23,17 +23,16 @@ import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.ml.math.functions.IgniteSupplier;
import org.apache.ignite.ml.nn.LossFunctions;
import org.apache.ignite.ml.nn.MultilayerPerceptron;
-import org.apache.ignite.ml.nn.updaters.ParameterUpdater;
-import org.apache.ignite.ml.nn.updaters.RPropUpdater;
-import org.apache.ignite.ml.nn.updaters.RPropUpdaterParams;
-import org.apache.ignite.ml.nn.updaters.UpdaterParams;
+import org.apache.ignite.ml.nn.updaters.ParameterUpdateCalculator;
+import org.apache.ignite.ml.nn.updaters.RPropParameterUpdate;
+import org.apache.ignite.ml.nn.updaters.RPropUpdateCalculator;
/**
* Local batch trainer for MLP.
*
* @param <P> Parameter updater parameters.
*/
-public class MLPLocalBatchTrainer<P extends UpdaterParams<? super MultilayerPerceptron>>
+public class MLPLocalBatchTrainer<P>
extends LocalBatchTrainer<MultilayerPerceptron, P> {
/**
* Default loss function.
@@ -51,7 +50,6 @@ public class MLPLocalBatchTrainer<P extends UpdaterParams<? super MultilayerPerc
*/
private static final int DEFAULT_MAX_ITERATIONS = 100;
-
/**
* Construct a trainer.
*
@@ -62,7 +60,7 @@ public class MLPLocalBatchTrainer<P extends UpdaterParams<? super MultilayerPerc
*/
public MLPLocalBatchTrainer(
IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss,
- IgniteSupplier<ParameterUpdater<? super MultilayerPerceptron, P>> updaterSupplier,
+ IgniteSupplier<ParameterUpdateCalculator<MultilayerPerceptron, P>> updaterSupplier,
double errorThreshold, int maxIterations) {
super(loss, updaterSupplier, errorThreshold, maxIterations);
}
@@ -72,7 +70,8 @@ public class MLPLocalBatchTrainer<P extends UpdaterParams<? super MultilayerPerc
*
* @return MLPLocalBatchTrainer with default parameters.
*/
- public static MLPLocalBatchTrainer<RPropUpdaterParams> getDefault() {
- return new MLPLocalBatchTrainer<>(DEFAULT_LOSS, RPropUpdater::new, DEFAULT_ERROR_THRESHOLD, DEFAULT_MAX_ITERATIONS);
+ public static MLPLocalBatchTrainer<RPropParameterUpdate> getDefault() {
+ return new MLPLocalBatchTrainer<>(DEFAULT_LOSS, () -> new RPropUpdateCalculator<>(), DEFAULT_ERROR_THRESHOLD,
+ DEFAULT_MAX_ITERATIONS);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/BaseSmoothParametrized.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/BaseSmoothParametrized.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/BaseSmoothParametrized.java
index b33c2c7..8e2f0df 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/BaseSmoothParametrized.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/BaseSmoothParametrized.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ml.nn.updaters;
+import org.apache.ignite.ml.Model;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.Vector;
import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
@@ -25,7 +26,7 @@ import org.apache.ignite.ml.math.functions.IgniteFunction;
/**
* Interface for models which are smooth functions of their parameters.
*/
-interface BaseSmoothParametrized<M extends BaseSmoothParametrized<M>> {
+interface BaseSmoothParametrized<M extends BaseSmoothParametrized<M> & Model<Matrix, Matrix>> {
/**
* Compose function in the following way: feed output of this model as input to second argument to loss function.
* After that we have a function g of three arguments: input, ground truth, parameters.
@@ -39,7 +40,8 @@ interface BaseSmoothParametrized<M extends BaseSmoothParametrized<M>> {
* @param truthBatch Batch of ground truths.
* @return Gradient of h at current point in parameters space.
*/
- Vector differentiateByParameters(IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss, Matrix inputsBatch, Matrix truthBatch);
+ Vector differentiateByParameters(IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss,
+ Matrix inputsBatch, Matrix truthBatch);
/**
* Get parameters vector.
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovParameterUpdate.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovParameterUpdate.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovParameterUpdate.java
new file mode 100644
index 0000000..8671285
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovParameterUpdate.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn.updaters;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Objects;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+
+/**
+ * Data needed for Nesterov parameters updater.
+ */
+public class NesterovParameterUpdate implements Serializable {
+ /**
+ * Previous step weights updates.
+ */
+ protected Vector prevIterationUpdates;
+
+ /**
+ * Construct NesterovParameterUpdate.
+ *
+ * @param paramsCnt Count of parameters on which updateCache happens.
+ */
+ public NesterovParameterUpdate(int paramsCnt) {
+ prevIterationUpdates = new DenseLocalOnHeapVector(paramsCnt).assign(0);
+ }
+
+ /**
+ * Construct NesterovParameterUpdate.
+ *
+ * @param prevIterationUpdates Previous iteration updates.
+ */
+ public NesterovParameterUpdate(Vector prevIterationUpdates) {
+ this.prevIterationUpdates = prevIterationUpdates;
+ }
+
+ /**
+ * Set previous step parameters updates.
+ *
+ * @param updates Parameters updates.
+ * @return This object with updated parameters updates.
+ */
+ public NesterovParameterUpdate setPreviousUpdates(Vector updates) {
+ prevIterationUpdates = updates;
+ return this;
+ }
+
+ /**
+ * Get previous step parameters updates.
+ *
+ * @return Previous step parameters updates.
+ */
+ public Vector prevIterationUpdates() {
+ return prevIterationUpdates;
+ }
+
+ /**
+ * Get sum of parameters updates.
+ *
+ * @param parameters Parameters to sum.
+ * @return Sum of parameters updates.
+ */
+ public static NesterovParameterUpdate sum(List<NesterovParameterUpdate> parameters) {
+ return parameters.stream().filter(Objects::nonNull).map(NesterovParameterUpdate::prevIterationUpdates)
+ .reduce(Vector::plus).map(NesterovParameterUpdate::new).orElse(null);
+ }
+
+ /**
+ * Get average of parameters updates.
+ *
+ * @param parameters Parameters to average.
+ * @return Average of parameters updates.
+ */
+ public static NesterovParameterUpdate avg(List<NesterovParameterUpdate> parameters) {
+ NesterovParameterUpdate sum = sum(parameters);
+ return sum != null ? sum.setPreviousUpdates(sum.prevIterationUpdates().divide(parameters.size())) : null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdateCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdateCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdateCalculator.java
new file mode 100644
index 0000000..5caddd4
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdateCalculator.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn.updaters;
+
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+
+/**
+ * Class encapsulating Nesterov algorithm for MLP parameters updateCache.
+ */
+public class NesterovUpdateCalculator<M extends SmoothParametrized>
+ implements ParameterUpdateCalculator<M, NesterovParameterUpdate> {
+ /**
+ * Learning rate.
+ */
+ private final double learningRate;
+
+ /**
+ * Loss function.
+ */
+ private IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
+
+ /**
+ * Momentum constant.
+ */
+ protected double momentum;
+
+ /**
+ * Construct NesterovUpdateCalculator.
+ *
+ * @param momentum Momentum constant.
+ */
+ public NesterovUpdateCalculator(double learningRate, double momentum) {
+ this.learningRate = learningRate;
+ this.momentum = momentum;
+ }
+
+ /** {@inheritDoc} */
+ @Override public NesterovParameterUpdate calculateNewUpdate(SmoothParametrized mdl,
+ NesterovParameterUpdate updaterParameters, int iteration, Matrix inputs, Matrix groundTruth) {
+ // TODO:IGNITE-7350 create new updateCache object here instead of in-place change.
+
+ if (iteration > 0) {
+ Vector curParams = mdl.parameters();
+ mdl.setParameters(curParams.minus(updaterParameters.prevIterationUpdates().times(momentum)));
+ }
+
+ Vector gradient = mdl.differentiateByParameters(loss, inputs, groundTruth);
+ updaterParameters.setPreviousUpdates(updaterParameters.prevIterationUpdates()
+ .plus(gradient.times(learningRate)));
+
+ return updaterParameters;
+ }
+
+ /** {@inheritDoc} */
+ @Override public NesterovParameterUpdate init(M mdl,
+ IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss) {
+ this.loss = loss;
+
+ return new NesterovParameterUpdate(mdl.parametersCount());
+ }
+
+ /** {@inheritDoc} */
+ @Override public <M1 extends M> M1 update(M1 obj, NesterovParameterUpdate update) {
+ Vector parameters = obj.parameters();
+ return (M1)obj.setParameters(parameters.minus(update.prevIterationUpdates()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdater.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdater.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdater.java
deleted file mode 100644
index 7b6a0c7..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdater.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-
-/**
- * Class encapsulating Nesterov algorithm for MLP parameters update.
- */
-public class NesterovUpdater implements ParameterUpdater<SmoothParametrized, NesterovUpdaterParams> {
- /**
- * Learning rate.
- */
- private final double learningRate;
-
- /**
- * Loss function.
- */
- private IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
-
- /**
- * Momentum constant.
- */
- protected double momentum;
-
- /**
- * Construct NesterovUpdater.
- *
- * @param momentum Momentum constant.
- */
- public NesterovUpdater(double learningRate, double momentum) {
- this.learningRate = learningRate;
- this.momentum = momentum;
- }
-
- /** {@inheritDoc} */
- @Override public NesterovUpdaterParams init(SmoothParametrized mdl,
- IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss) {
- this.loss = loss;
-
- return new NesterovUpdaterParams(mdl.parametersCount());
- }
-
- /** {@inheritDoc} */
- @Override public NesterovUpdaterParams updateParams(SmoothParametrized mdl, NesterovUpdaterParams updaterParameters,
- int iteration, Matrix inputs, Matrix groundTruth) {
-
- if (iteration > 0) {
- Vector curParams = mdl.parameters();
- mdl.setParameters(curParams.minus(updaterParameters.prevIterationUpdates().times(momentum)));
- }
-
- Vector gradient = mdl.differentiateByParameters(loss, inputs, groundTruth);
- updaterParameters.setPreviousUpdates(updaterParameters.prevIterationUpdates().plus(gradient.times(learningRate)));
-
- return updaterParameters;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdaterParams.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdaterParams.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdaterParams.java
deleted file mode 100644
index d403ea1..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdaterParams.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-
-/**
- * Data needed for Nesterov parameters updater.
- */
-public class NesterovUpdaterParams implements UpdaterParams<SmoothParametrized> {
- /**
- * Previous step weights updates.
- */
- protected Vector prevIterationUpdates;
-
- /**
- * Construct NesterovUpdaterParams.
- *
- * @param paramsCnt Count of parameters on which update happens.
- */
- public NesterovUpdaterParams(int paramsCnt) {
- prevIterationUpdates = new DenseLocalOnHeapVector(paramsCnt).assign(0);
- }
-
- /**
- * Set previous step parameters updates.
- *
- * @param updates Parameters updates.
- * @return This object with updated parameters updates.
- */
- public NesterovUpdaterParams setPreviousUpdates(Vector updates) {
- prevIterationUpdates = updates;
- return this;
- }
-
- /**
- * Get previous step parameters updates.
- *
- * @return Previous step parameters updates.
- */
- public Vector prevIterationUpdates() {
- return prevIterationUpdates;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public <M extends SmoothParametrized> M update(M obj) {
- Vector parameters = obj.parameters();
- return (M)obj.setParameters(parameters.minus(prevIterationUpdates));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/ParameterUpdateCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/ParameterUpdateCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/ParameterUpdateCalculator.java
new file mode 100644
index 0000000..77e3763
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/ParameterUpdateCalculator.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn.updaters;
+
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+
+/**
+ * Interface for classes encapsulating parameters updateCache logic.
+ *
+ * @param <M> Type of model to be updated.
+ * @param <P> Type of parameters needed for this updater.
+ */
+public interface ParameterUpdateCalculator<M, P> {
+ /**
+ * Initializes the updater.
+ *
+ * @param mdl Model to be trained.
+ * @param loss Loss function.
+ */
+ P init(M mdl, IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss);
+
+ /**
+ * Calculate new updateCache.
+ *
+ * @param mdl Model to be updated.
+ * @param updaterParameters Updater parameters to updateCache.
+ * @param iteration Current trainer iteration.
+ * @param inputs Inputs.
+ * @param groundTruth True values.
+ * @return Updated parameters.
+ */
+ P calculateNewUpdate(M mdl, P updaterParameters, int iteration, Matrix inputs, Matrix groundTruth);
+
+ /**
+ * Update given obj with this parameters.
+ *
+ * @param obj Object to be updated.
+ */
+ <M1 extends M> M1 update(M1 obj, P update);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/ParameterUpdater.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/ParameterUpdater.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/ParameterUpdater.java
deleted file mode 100644
index e8e28fd..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/ParameterUpdater.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-
-/**
- * Interface for classes encapsulating parameters update logic.
- *
- * @param <M> Type of model to be updated.
- * @param <P> Type of parameters needed for this updater.
- */
-public interface ParameterUpdater<M, P extends UpdaterParams> {
- /**
- * Initializes the updater.
- *
- * @param mdl Model to be trained.
- * @param loss Loss function.
- */
- P init(M mdl, IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss);
-
- /**
- * Update updater parameters.
- *
- * @param mdl Model to be updated.
- * @param updaterParameters Updater parameters to update.
- * @param iteration Current trainer iteration.
- * @param inputs Inputs.
- * @param groundTruth True values.
- * @return Updated parameters.
- */
- P updateParams(M mdl, P updaterParameters, int iteration, Matrix inputs, Matrix groundTruth);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropParameterUpdate.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropParameterUpdate.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropParameterUpdate.java
new file mode 100644
index 0000000..e2fa4d5
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropParameterUpdate.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn.updaters;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.VectorUtils;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+
+/**
+ * Data needed for RProp updater.
+ * <p>
+ * See <a href="https://paginas.fe.up.pt/~ee02162/dissertacao/RPROP%20paper.pdf">RProp</a>.</p>
+ */
+public class RPropParameterUpdate implements Serializable {
+ /**
+ * Previous iteration parameters updates. In original paper they are labeled with "delta w".
+ */
+ protected Vector prevIterationUpdates;
+
+ /**
+ * Previous iteration model partial derivatives by parameters.
+ */
+ protected Vector prevIterationGradient;
+ /**
+ * Previous iteration parameters deltas. In original paper they are labeled with "delta".
+ */
+ protected Vector deltas;
+
+ /**
+ * Updates mask (values by which updateCache is multiplied).
+ */
+ protected Vector updatesMask;
+
+ /**
+ * Construct RPropParameterUpdate.
+ *
+ * @param paramsCnt Parameters count.
+ * @param initUpdate Initial updateCache (in original work labeled as "delta_0").
+ */
+ RPropParameterUpdate(int paramsCnt, double initUpdate) {
+ prevIterationUpdates = new DenseLocalOnHeapVector(paramsCnt);
+ prevIterationGradient = new DenseLocalOnHeapVector(paramsCnt);
+ deltas = new DenseLocalOnHeapVector(paramsCnt).assign(initUpdate);
+ updatesMask = new DenseLocalOnHeapVector(paramsCnt);
+ }
+
+ /**
+ * Construct instance of this class by given parameters.
+ *
+ * @param prevIterationUpdates Previous iteration parameters updates.
+ * @param prevIterationGradient Previous iteration model partial derivatives by parameters.
+ * @param deltas Previous iteration parameters deltas.
+ * @param updatesMask Updates mask.
+ */
+ public RPropParameterUpdate(Vector prevIterationUpdates, Vector prevIterationGradient,
+ Vector deltas, Vector updatesMask) {
+ this.prevIterationUpdates = prevIterationUpdates;
+ this.prevIterationGradient = prevIterationGradient;
+ this.deltas = deltas;
+ this.updatesMask = updatesMask;
+ }
+
+ /**
+ * Get bias deltas.
+ *
+ * @return Bias deltas.
+ */
+ Vector deltas() {
+ return deltas;
+ }
+
+ /**
+ * Get previous iteration biases updates. In original paper they are labeled with "delta w".
+ *
+ * @return Biases updates.
+ */
+ Vector prevIterationUpdates() {
+ return prevIterationUpdates;
+ }
+
+ /**
+ * Set previous iteration parameters updates. In original paper they are labeled with "delta w".
+ *
+ * @param updates New parameters updates value.
+ * @return This object.
+ */
+ private RPropParameterUpdate setPrevIterationUpdates(Vector updates) {
+ prevIterationUpdates = updates;
+
+ return this;
+ }
+
+ /**
+ * Get previous iteration loss function partial derivatives by parameters.
+ *
+ * @return Previous iteration loss function partial derivatives by parameters.
+ */
+ Vector prevIterationGradient() {
+ return prevIterationGradient;
+ }
+
+ /**
+ * Set previous iteration loss function partial derivatives by parameters.
+ *
+ * @return This object.
+ */
+ private RPropParameterUpdate setPrevIterationGradient(Vector gradient) {
+ prevIterationGradient = gradient;
+ return this;
+ }
+
+ /**
+ * Get updates mask (values by which updateCache is multiplied).
+ *
+ * @return Updates mask (values by which updateCache is multiplied).
+ */
+ public Vector updatesMask() {
+ return updatesMask;
+ }
+
+ /**
+ * Set updates mask (values by which updateCache is multiplied).
+ *
+ * @param updatesMask New updatesMask.
+ * @return This object.
+ */
+ public RPropParameterUpdate setUpdatesMask(Vector updatesMask) {
+ this.updatesMask = updatesMask;
+
+ return this;
+ }
+
+ /**
+ * Set previous iteration deltas.
+ *
+ * @param deltas New deltas.
+ * @return This object.
+ */
+ public RPropParameterUpdate setDeltas(Vector deltas) {
+ this.deltas = deltas;
+
+ return this;
+ }
+
+ /**
+ * Sums updates during one training.
+ *
+ * @param updates Updates.
+ * @return Sum of updates during one training.
+ */
+ public static RPropParameterUpdate sumLocal(List<RPropParameterUpdate> updates) {
+ List<RPropParameterUpdate> nonNullUpdates = updates.stream().filter(Objects::nonNull)
+ .collect(Collectors.toList());
+
+ if (nonNullUpdates.isEmpty())
+ return null;
+
+ Vector newDeltas = nonNullUpdates.get(nonNullUpdates.size() - 1).deltas();
+ Vector newGradient = nonNullUpdates.get(nonNullUpdates.size() - 1).prevIterationGradient();
+ Vector totalUpdate = nonNullUpdates.stream().map(pu -> VectorUtils.elementWiseTimes(pu.updatesMask().copy(),
+ pu.prevIterationUpdates())).reduce(Vector::plus).orElse(null);
+
+ return new RPropParameterUpdate(totalUpdate, newGradient, newDeltas,
+ new DenseLocalOnHeapVector(newDeltas.size()).assign(1.0));
+ }
+
+ /**
+ * Sums updates returned by different trainings.
+ *
+ * @param updates Updates.
+ * @return Sum of updates during returned by different trainings.
+ */
+ public static RPropParameterUpdate sum(List<RPropParameterUpdate> updates) {
+ Vector totalUpdate = updates.stream().filter(Objects::nonNull)
+ .map(pu -> VectorUtils.elementWiseTimes(pu.updatesMask().copy(), pu.prevIterationUpdates()))
+ .reduce(Vector::plus).orElse(null);
+ Vector totalDelta = updates.stream().filter(Objects::nonNull)
+ .map(RPropParameterUpdate::deltas).reduce(Vector::plus).orElse(null);
+ Vector totalGradient = updates.stream().filter(Objects::nonNull)
+ .map(RPropParameterUpdate::prevIterationGradient).reduce(Vector::plus).orElse(null);
+
+ if (totalUpdate != null)
+ return new RPropParameterUpdate(totalUpdate, totalGradient, totalDelta,
+ new DenseLocalOnHeapVector(Objects.requireNonNull(totalDelta).size()).assign(1.0));
+
+ return null;
+ }
+
+ /**
+ * Averages updates returned by different trainings.
+ *
+ * @param updates Updates.
+ * @return Averages of updates during returned by different trainings.
+ */
+ public static RPropParameterUpdate avg(List<RPropParameterUpdate> updates) {
+ List<RPropParameterUpdate> nonNullUpdates = updates.stream()
+ .filter(Objects::nonNull).collect(Collectors.toList());
+ int size = nonNullUpdates.size();
+
+ RPropParameterUpdate sum = sum(updates);
+ if (sum != null)
+ return sum.
+ setPrevIterationGradient(sum.prevIterationGradient().divide(size)).
+ setPrevIterationUpdates(sum.prevIterationUpdates().divide(size)).
+ setDeltas(sum.deltas().divide(size));
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdateCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdateCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdateCalculator.java
new file mode 100644
index 0000000..99f39c9
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdateCalculator.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn.updaters;
+
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.VectorUtils;
+import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.util.MatrixUtil;
+
+/**
+ * Class encapsulating RProp algorithm.
+ * <p>
+ * See <a href="https://paginas.fe.up.pt/~ee02162/dissertacao/RPROP%20paper.pdf">RProp</a>.</p>
+ */
+public class RPropUpdateCalculator<M extends SmoothParametrized> implements ParameterUpdateCalculator<M, RPropParameterUpdate> {
+ /**
+ * Default initial update.
+ */
+ private static double DFLT_INIT_UPDATE = 0.1;
+
+ /**
+ * Default acceleration rate.
+ */
+ private static double DFLT_ACCELERATION_RATE = 1.2;
+
+ /**
+ * Default deacceleration rate.
+ */
+ private static double DFLT_DEACCELERATION_RATE = 0.5;
+
+ /**
+ * Initial update.
+ */
+ private final double initUpdate;
+
+ /**
+ * Acceleration rate.
+ */
+ private final double accelerationRate;
+
+ /**
+ * Deacceleration rate.
+ */
+ private final double deaccelerationRate;
+
+ /**
+ * Maximal value for update.
+ */
+ private final static double UPDATE_MAX = 50.0;
+
+ /**
+ * Minimal value for update.
+ */
+ private final static double UPDATE_MIN = 1E-6;
+
+ /**
+ * Loss function.
+ */
+ protected IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
+
+ /**
+ * Construct RPropUpdateCalculator.
+ *
+ * @param initUpdate Initial update.
+ * @param accelerationRate Acceleration rate.
+ * @param deaccelerationRate Deacceleration rate.
+ */
+ public RPropUpdateCalculator(double initUpdate, double accelerationRate, double deaccelerationRate) {
+ this.initUpdate = initUpdate;
+ this.accelerationRate = accelerationRate;
+ this.deaccelerationRate = deaccelerationRate;
+ }
+
+ /**
+ * Construct RPropUpdateCalculator with default parameters.
+ */
+ public RPropUpdateCalculator() {
+ this(DFLT_INIT_UPDATE, DFLT_ACCELERATION_RATE, DFLT_DEACCELERATION_RATE);
+ }
+
+ /** {@inheritDoc} */
+ @Override public RPropParameterUpdate calculateNewUpdate(SmoothParametrized mdl, RPropParameterUpdate updaterParams,
+ int iteration, Matrix inputs, Matrix groundTruth) {
+ Vector gradient = mdl.differentiateByParameters(loss, inputs, groundTruth);
+ Vector prevGradient = updaterParams.prevIterationGradient();
+ Vector derSigns;
+
+ if (prevGradient != null)
+ derSigns = VectorUtils.zipWith(prevGradient, gradient, (x, y) -> Math.signum(x * y));
+ else
+ derSigns = gradient.like(gradient.size()).assign(1.0);
+
+ Vector newDeltas = updaterParams.deltas().copy().map(derSigns, (prevDelta, sign) -> {
+ if (sign > 0)
+ return Math.min(prevDelta * accelerationRate, UPDATE_MAX);
+ else if (sign < 0)
+ return Math.max(prevDelta * deaccelerationRate, UPDATE_MIN);
+ else
+ return prevDelta;
+ });
+
+ Vector newPrevIterationUpdates = MatrixUtil.zipWith(gradient, updaterParams.deltas(), (der, delta, i) -> {
+ if (derSigns.getX(i) >= 0)
+ return -Math.signum(der) * delta;
+
+ return updaterParams.prevIterationUpdates().getX(i);
+ });
+
+ Vector updatesMask = MatrixUtil.zipWith(derSigns, updaterParams.prevIterationUpdates(), (sign, upd, i) -> {
+ if (sign < 0)
+ gradient.setX(i, 0.0);
+
+ if (sign >= 0)
+ return 1.0;
+ else
+ return -1.0;
+ });
+
+ return new RPropParameterUpdate(newPrevIterationUpdates, gradient.copy(), newDeltas, updatesMask);
+ }
+
+ /** {@inheritDoc} */
+ @Override public RPropParameterUpdate init(M mdl,
+ IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss) {
+ this.loss = loss;
+ return new RPropParameterUpdate(mdl.parametersCount(), initUpdate);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <M1 extends M> M1 update(M1 obj, RPropParameterUpdate update) {
+ Vector updatesToAdd = VectorUtils.elementWiseTimes(update.updatesMask().copy(), update.prevIterationUpdates());
+ return (M1)obj.setParameters(obj.parameters().plus(updatesToAdd));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdater.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdater.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdater.java
deleted file mode 100644
index 37963b4..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdater.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.VectorUtils;
-import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.util.MatrixUtil;
-
-/**
- * Class encapsulating RProp algorithm.
- * <p>
- * See <a href="https://paginas.fe.up.pt/~ee02162/dissertacao/RPROP%20paper.pdf">RProp</a>.</p>
- */
-public class RPropUpdater implements ParameterUpdater<SmoothParametrized, RPropUpdaterParams> {
- /**
- * Default initial update.
- */
- private static double DFLT_INIT_UPDATE = 0.1;
-
- /**
- * Default acceleration rate.
- */
- private static double DFLT_ACCELERATION_RATE = 1.2;
-
- /**
- * Default deacceleration rate.
- */
- private static double DFLT_DEACCELERATION_RATE = 0.5;
-
- /**
- * Initial update.
- */
- private final double initUpdate;
-
- /**
- * Acceleration rate.
- */
- private final double accelerationRate;
-
- /**
- * Deacceleration rate.
- */
- private final double deaccelerationRate;
-
- /**
- * Maximal value for update.
- */
- private final static double UPDATE_MAX = 50.0;
-
- /**
- * Minimal value for update.
- */
- private final static double UPDATE_MIN = 1E-6;
-
- /**
- * Loss function.
- */
- protected IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
-
- /**
- * Construct RPropUpdater.
- *
- * @param initUpdate Initial update.
- * @param accelerationRate Acceleration rate.
- * @param deaccelerationRate Deacceleration rate.
- */
- public RPropUpdater(double initUpdate, double accelerationRate, double deaccelerationRate) {
- this.initUpdate = initUpdate;
- this.accelerationRate = accelerationRate;
- this.deaccelerationRate = deaccelerationRate;
- }
-
- /**
- * Construct RPropUpdater with default parameters.
- */
- public RPropUpdater() {
- this(DFLT_INIT_UPDATE, DFLT_ACCELERATION_RATE, DFLT_DEACCELERATION_RATE);
- }
-
- /** {@inheritDoc} */
- @Override public RPropUpdaterParams init(SmoothParametrized mdl,
- IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss) {
- this.loss = loss;
- return new RPropUpdaterParams(mdl.parametersCount(), initUpdate);
- }
-
- /** {@inheritDoc} */
- @Override public RPropUpdaterParams updateParams(SmoothParametrized mdl, RPropUpdaterParams updaterParams,
- int iteration, Matrix inputs, Matrix groundTruth) {
- Vector gradient = mdl.differentiateByParameters(loss, inputs, groundTruth);
- Vector prevGradient = updaterParams.prevIterationGradient();
- Vector derSigns;
-
- if (prevGradient != null)
- derSigns = VectorUtils.zipWith(prevGradient, gradient, (x, y) -> Math.signum(x * y));
- else
- derSigns = gradient.like(gradient.size()).assign(1.0);
-
- updaterParams.deltas().map(derSigns, (prevDelta, sign) -> {
- if (sign > 0)
- return Math.min(prevDelta * accelerationRate, UPDATE_MAX);
- else if (sign < 0)
- return Math.max(prevDelta * deaccelerationRate, UPDATE_MIN);
- else
- return prevDelta;
- });
-
- updaterParams.setPrevIterationBiasesUpdates(MatrixUtil.zipWith(gradient, updaterParams.deltas(), (der, delta, i) -> {
- if (derSigns.getX(i) >= 0)
- return -Math.signum(der) * delta;
-
- return updaterParams.prevIterationUpdates().getX(i);
- }));
-
- Vector updatesMask = MatrixUtil.zipWith(derSigns, updaterParams.prevIterationUpdates(), (sign, upd, i) -> {
- if (sign < 0)
- gradient.setX(i, 0.0);
-
- if (sign >= 0)
- return 1.0;
- else
- return -1.0;
- });
-
- updaterParams.setUpdatesMask(updatesMask);
- updaterParams.setPrevIterationWeightsDerivatives(gradient.copy());
-
- return updaterParams;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdaterParams.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdaterParams.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdaterParams.java
deleted file mode 100644
index 080e809..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdaterParams.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.VectorUtils;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-
-/**
- * Data needed for RProp updater.
- * <p>
- * See <a href="https://paginas.fe.up.pt/~ee02162/dissertacao/RPROP%20paper.pdf">RProp</a>.</p>
- */
-public class RPropUpdaterParams implements UpdaterParams<SmoothParametrized> {
- /**
- * Previous iteration weights updates. In original paper they are labeled with "delta w".
- */
- protected Vector prevIterationUpdates;
-
- /**
- * Previous iteration model partial derivatives by parameters.
- */
- protected Vector prevIterationGradient;
- /**
- * Previous iteration parameters deltas. In original paper they are labeled with "delta".
- */
- protected Vector deltas;
-
- /**
- * Updates mask (values by which update is multiplied).
- */
- protected Vector updatesMask;
-
- /**
- * Construct RPropUpdaterParams.
- *
- * @param paramsCnt Parameters count.
- * @param initUpdate Initial update (in original work labeled as "delta_0").
- */
- RPropUpdaterParams(int paramsCnt, double initUpdate) {
- prevIterationUpdates = new DenseLocalOnHeapVector(paramsCnt);
- prevIterationGradient = new DenseLocalOnHeapVector(paramsCnt);
- deltas = new DenseLocalOnHeapVector(paramsCnt).assign(initUpdate);
- updatesMask = new DenseLocalOnHeapVector(paramsCnt);
- }
-
- /**
- * Get bias deltas.
- *
- * @return Bias deltas.
- */
- Vector deltas() {
- return deltas;
- }
-
- /**
- * Get previous iteration biases updates. In original paper they are labeled with "delta w".
- *
- * @return Biases updates.
- */
- Vector prevIterationUpdates() {
- return prevIterationUpdates;
- }
-
- /**
- * Set previous iteration parameters updates. In original paper they are labeled with "delta w".
- *
- * @param updates New parameters updates value.
- * @return This object.
- */
- Vector setPrevIterationBiasesUpdates(Vector updates) {
- return prevIterationUpdates = updates;
- }
-
- /**
- * Get previous iteration loss function partial derivatives by parameters.
- *
- * @return Previous iteration loss function partial derivatives by parameters.
- */
- Vector prevIterationGradient() {
- return prevIterationGradient;
- }
-
- /**
- * Set previous iteration loss function partial derivatives by parameters.
- *
- * @return This object.
- */
- RPropUpdaterParams setPrevIterationWeightsDerivatives(Vector gradient) {
- prevIterationGradient = gradient;
- return this;
- }
-
- /**
- * Get updates mask (values by which update is multiplied).
- *
- * @return Updates mask (values by which update is multiplied).
- */
- public Vector updatesMask() {
- return updatesMask;
- }
-
- /**
- * Set updates mask (values by which update is multiplied).
- *
- * @param updatesMask New updatesMask.
- */
- public RPropUpdaterParams setUpdatesMask(Vector updatesMask) {
- this.updatesMask = updatesMask;
-
- return this;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public <M extends SmoothParametrized> M update(M obj) {
- Vector updatesToAdd = VectorUtils.elementWiseTimes(updatesMask.copy(), prevIterationUpdates);
- return (M)obj.setParameters(obj.parameters().plus(updatesToAdd));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDParameter.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDParameter.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDParameter.java
new file mode 100644
index 0000000..7159621
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDParameter.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn.updaters;
+
+import java.io.Serializable;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+
+/**
+ * Parameters for {@link SimpleGDUpdateCalculator}.
+ */
+public class SimpleGDParameter implements Serializable {
+ /**
+ * Gradient.
+ */
+ private Vector gradient;
+
+ /**
+ * Learning rate.
+ */
+ private double learningRate;
+
+ /**
+ * Construct instance of this class.
+ *
+ * @param paramsCnt Count of parameters.
+ * @param learningRate Learning rate.
+ */
+ public SimpleGDParameter(int paramsCnt, double learningRate) {
+ gradient = new DenseLocalOnHeapVector(paramsCnt);
+ this.learningRate = learningRate;
+ }
+
+ /**
+ * Construct instance of this class.
+ *
+ * @param gradient Gradient.
+ * @param learningRate Learning rate.
+ */
+ public SimpleGDParameter(Vector gradient, double learningRate) {
+ this.gradient = gradient;
+ this.learningRate = learningRate;
+ }
+
+ /**
+ * Get gradient.
+ *
+ * @return Get gradient.
+ */
+ public Vector gradient() {
+ return gradient;
+ }
+
+ /**
+ * Get learning rate.
+ *
+ * @return learning rate.
+ */
+ public double learningRate() {
+ return learningRate;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDParams.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDParams.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDParams.java
deleted file mode 100644
index 50a120a..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDParams.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-
-/**
- * Parameters for {@link SimpleGDUpdater}.
- */
-public class SimpleGDParams implements UpdaterParams<SmoothParametrized> {
- /**
- * Gradient.
- */
- private Vector gradient;
-
- /**
- * Learning rate.
- */
- private double learningRate;
-
- /**
- * Construct instance of this class.
- *
- * @param paramsCnt Count of parameters.
- * @param learningRate Learning rate.
- */
- public SimpleGDParams(int paramsCnt, double learningRate) {
- gradient = new DenseLocalOnHeapVector(paramsCnt);
- this.learningRate = learningRate;
- }
-
- /**
- * Construct instance of this class.
- *
- * @param gradient Gradient.
- * @param learningRate Learning rate.
- */
- public SimpleGDParams(Vector gradient, double learningRate) {
- this.gradient = gradient;
- this.learningRate = learningRate;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public <M extends SmoothParametrized> M update(M obj) {
- Vector params = obj.parameters();
- return (M)obj.setParameters(params.minus(gradient.times(learningRate)));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDUpdateCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDUpdateCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDUpdateCalculator.java
new file mode 100644
index 0000000..d2197d9
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDUpdateCalculator.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn.updaters;
+
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+
+/**
+ * Simple gradient descent parameters updater.
+ */
+public class SimpleGDUpdateCalculator<M extends SmoothParametrized> implements ParameterUpdateCalculator<M, SimpleGDParameter> {
+ /**
+ * Learning rate.
+ */
+ private double learningRate;
+
+ /**
+ * Loss function.
+ */
+ protected IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
+
+ /**
+ * Construct SimpleGDUpdateCalculator.
+ *
+ * @param learningRate Learning rate.
+ */
+ public SimpleGDUpdateCalculator(double learningRate) {
+ this.learningRate = learningRate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public SimpleGDParameter init(M mdl,
+ IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss) {
+ this.loss = loss;
+ return new SimpleGDParameter(mdl.parametersCount(), learningRate);
+ }
+
+ /** {@inheritDoc} */
+ @Override public SimpleGDParameter calculateNewUpdate(SmoothParametrized mlp, SimpleGDParameter updaterParameters,
+ int iteration, Matrix inputs, Matrix groundTruth) {
+ return new SimpleGDParameter(mlp.differentiateByParameters(loss, inputs, groundTruth), learningRate);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <M1 extends M> M1 update(M1 obj, SimpleGDParameter update) {
+ Vector params = obj.parameters();
+ return (M1)obj.setParameters(params.minus(update.gradient().times(learningRate)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDUpdater.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDUpdater.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDUpdater.java
deleted file mode 100644
index 5bf9c3f..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDUpdater.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-
-/**
- * Simple gradient descent parameters updater.
- */
-public class SimpleGDUpdater implements ParameterUpdater<SmoothParametrized, SimpleGDParams> {
- /**
- * Learning rate.
- */
- private double learningRate;
-
- /**
- * Loss function.
- */
- protected IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
-
- /**
- * Construct SimpleGDUpdater.
- *
- * @param learningRate Learning rate.
- */
- public SimpleGDUpdater(double learningRate) {
- this.learningRate = learningRate;
- }
-
- /** {@inheritDoc} */
- @Override public SimpleGDParams init(SmoothParametrized mlp,
- IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss) {
- this.loss = loss;
- return new SimpleGDParams(mlp.parametersCount(), learningRate);
- }
-
- /** {@inheritDoc} */
- @Override public SimpleGDParams updateParams(SmoothParametrized mlp, SimpleGDParams updaterParameters,
- int iteration, Matrix inputs, Matrix groundTruth) {
- return new SimpleGDParams(mlp.differentiateByParameters(loss, inputs, groundTruth), learningRate);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SmoothParametrized.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SmoothParametrized.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SmoothParametrized.java
index 5c4f59f..1534a6d 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SmoothParametrized.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SmoothParametrized.java
@@ -17,8 +17,11 @@
package org.apache.ignite.ml.nn.updaters;
+import org.apache.ignite.ml.Model;
+import org.apache.ignite.ml.math.Matrix;
+
/**
* Interface for models which are smooth functions of their parameters.
*/
-public interface SmoothParametrized<M extends SmoothParametrized<M>> extends BaseSmoothParametrized<M> {
+public interface SmoothParametrized<M extends SmoothParametrized<M>> extends BaseSmoothParametrized<M>, Model<Matrix, Matrix> {
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/UpdaterParams.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/UpdaterParams.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/UpdaterParams.java
deleted file mode 100644
index cd5bc32..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/UpdaterParams.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-/**
- * A common interface for parameter updaters.
- *
- * @param <T> Type of object to be updated with this params.
- */
-public interface UpdaterParams<T> {
- /**
- * Update given obj with this parameters.
- *
- * @param obj Object to be updated.
- */
- <M extends T> M update(M obj);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/trainers/Trainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/Trainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/Trainer.java
new file mode 100644
index 0000000..7540d6f
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/Trainer.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.trainers;
+
+import org.apache.ignite.ml.Model;
+
+/** Trainer interface. */
+public interface Trainer<M extends Model, T> {
+ /** Train the model based on provided data.
+ *
+ * @param data Data for training.
+ * @return Trained model.
+ */
+ public M train(T data);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/BaseLocalProcessorJob.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/BaseLocalProcessorJob.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/BaseLocalProcessorJob.java
new file mode 100644
index 0000000..d70b3f1
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/BaseLocalProcessorJob.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.trainers.group;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.functions.IgniteSupplier;
+
+/**
+ * Base job for group training.
+ * It's purpose is to apply worker to each element (cache key or cache entry) of given cache specified
+ * by keySupplier. Worker produces {@link ResultAndUpdates} object which contains 'side effects' which are updates
+ * needed to apply to caches and computation result.
+ * After we get all {@link ResultAndUpdates} we merge all 'update' parts of them for each node
+ * and apply them on corresponding node, also we reduce all 'result' by some given reducer.
+ *
+ * @param <K> Type of keys of cache used for group trainer.
+ * @param <V> Type of values of cache used for group trainer.
+ * @param <T> Type of elements to which workers are applier.
+ * @param <R> Type of result of worker.
+ */
+public abstract class BaseLocalProcessorJob<K, V, T, R extends Serializable> implements ComputeJob {
+ /**
+ * UUID of group training.
+ */
+ protected UUID trainingUUID;
+
+ /**
+ * Worker.
+ */
+ protected IgniteFunction<T, ResultAndUpdates<R>> worker;
+
+ /**
+ * Supplier of keys determining elements to which worker should be applied.
+ */
+ protected IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> keySupplier;
+
+ /**
+ * Operator used to reduce results from worker.
+ */
+ protected IgniteFunction<List<R>, R> reducer;
+
+ /**
+ * Name of cache used for training.
+ */
+ protected String cacheName;
+
+ /**
+ * Construct instance of this class with given arguments.
+ *
+ * @param worker Worker.
+ * @param keySupplier Supplier of keys.
+ * @param reducer Reducer.
+ * @param trainingUUID UUID of training.
+ * @param cacheName Name of cache used for training.
+ */
+ public BaseLocalProcessorJob(
+ IgniteFunction<T, ResultAndUpdates<R>> worker,
+ IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> keySupplier,
+ IgniteFunction<List<R>, R> reducer,
+ UUID trainingUUID, String cacheName) {
+ this.worker = worker;
+ this.keySupplier = keySupplier;
+ this.reducer = reducer;
+ this.trainingUUID = trainingUUID;
+ this.cacheName = cacheName;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancel() {
+ // NO-OP.
+ }
+
+ /** {@inheritDoc} */
+ @Override public R execute() throws IgniteException {
+ List<ResultAndUpdates<R>> resultsAndUpdates = toProcess().
+ map(worker).
+ collect(Collectors.toList());
+
+ ResultAndUpdates<R> totalRes = ResultAndUpdates.sum(reducer, resultsAndUpdates);
+
+ totalRes.applyUpdates(ignite());
+
+ return totalRes.result();
+ }
+
+ /**
+ * Get stream of elements to process.
+ *
+ * @return Stream of elements to process.
+ */
+ protected abstract Stream<T> toProcess();
+
+ /**
+ * Ignite instance.
+ *
+ * @return Ignite instance.
+ */
+ protected static Ignite ignite() {
+ return Ignition.localIgnite();
+ }
+
+ /**
+ * Get cache used for training.
+ *
+ * @return Cache used for training.
+ */
+ protected IgniteCache<GroupTrainerCacheKey<K>, V> cache() {
+ return ignite().getOrCreateCache(cacheName);
+ }
+
+ /**
+ * Get affinity function for cache used in group training.
+ *
+ * @return Affinity function for cache used in group training.
+ */
+ protected Affinity<GroupTrainerCacheKey> affinity() {
+ return ignite().affinity(cacheName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e24927e/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ConstModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ConstModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ConstModel.java
new file mode 100644
index 0000000..75f8179
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ConstModel.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.trainers.group;
+
+import org.apache.ignite.ml.Model;
+
+/**
+ * Model which outputs given constant.
+ *
+ * @param <T> Type of constant.
+ */
+public class ConstModel<T> implements Model<T, T> {
+ /**
+ * Constant to be returned by this model.
+ */
+ private T c;
+
+ /**
+ * Create instance of this class specified by input parameters.
+ *
+ * @param c Constant to be returned by this model.
+ */
+ public ConstModel(T c) {
+ this.c = c;
+ }
+
+ /** {@inheritDoc} */
+ @Override public T apply(T val) {
+ return c;
+ }
+}