You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ak...@apache.org on 2018/01/17 12:23:03 UTC
[05/43] ignite git commit: IGNITE-7350: Distributed MLP
cleanup/refactoring
IGNITE-7350: Distributed MLP cleanup/refactoring
this closes #3368
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6eccf230
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6eccf230
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6eccf230
Branch: refs/heads/ignite-7274
Commit: 6eccf23009c1e0a03638255555e8c93a77667126
Parents: 696277d
Author: artemmalykh <am...@gridgain.com>
Authored: Tue Jan 16 13:29:04 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Tue Jan 16 13:29:04 2018 +0300
----------------------------------------------------------------------
.../examples/ml/nn/MLPGroupTrainerExample.java | 2 +-
.../examples/ml/nn/MLPLocalTrainerExample.java | 6 +-
.../ignite/ml/nn/LocalBatchTrainerInput.java | 42 ----
.../org/apache/ignite/ml/nn/LossFunctions.java | 47 ----
.../ml/nn/MLPGroupUpdateTrainerCacheInput.java | 28 ++-
.../ignite/ml/nn/MultilayerPerceptron.java | 2 +-
.../AbstractMLPGroupUpdateTrainerInput.java | 2 +-
.../distributed/MLPGroupUpdateTrainer.java | 11 +-
.../MLPGroupUpdateTrainerDataCache.java | 2 +-
.../distributed/MLPGroupUpdateTrainingData.java | 29 ++-
.../MLPGroupUpdateTrainingLoopData.java | 30 ++-
.../trainers/distributed/MLPMetaoptimizer.java | 2 +-
.../ml/nn/trainers/local/LocalBatchTrainer.java | 179 ---------------
.../nn/trainers/local/MLPLocalBatchTrainer.java | 9 +-
.../ml/nn/updaters/BaseSmoothParametrized.java | 66 ------
.../ml/nn/updaters/NesterovParameterUpdate.java | 94 --------
.../nn/updaters/NesterovUpdateCalculator.java | 85 -------
.../nn/updaters/ParameterUpdateCalculator.java | 58 -----
.../ml/nn/updaters/RPropParameterUpdate.java | 228 -------------------
.../ml/nn/updaters/RPropUpdateCalculator.java | 151 ------------
.../ml/nn/updaters/SimpleGDParameter.java | 77 -------
.../nn/updaters/SimpleGDUpdateCalculator.java | 66 ------
.../ml/nn/updaters/SmoothParametrized.java | 27 ---
.../ignite/ml/nn/updaters/package-info.java | 22 --
.../ml/optimization/BarzilaiBorweinUpdater.java | 8 +-
.../ml/optimization/BaseParametrized.java | 58 +++++
.../ignite/ml/optimization/GradientDescent.java | 3 +-
.../ml/optimization/GradientFunction.java | 2 +-
.../ignite/ml/optimization/LossFunctions.java | 47 ++++
.../ignite/ml/optimization/Parametrized.java | 26 +++
.../ignite/ml/optimization/SimpleUpdater.java | 3 +-
.../ml/optimization/SmoothParametrized.java | 45 ++++
.../NesterovParameterUpdate.java | 94 ++++++++
.../NesterovUpdateCalculator.java | 86 +++++++
.../ParameterUpdateCalculator.java | 59 +++++
.../updatecalculators/RPropParameterUpdate.java | 228 +++++++++++++++++++
.../RPropUpdateCalculator.java | 152 +++++++++++++
.../updatecalculators/SimpleGDParameter.java | 77 +++++++
.../SimpleGDUpdateCalculator.java | 67 ++++++
.../updatecalculators/package-info.java | 22 ++
.../util/SparseDistributedMatrixMapReducer.java | 2 +-
.../org/apache/ignite/ml/trainers/Trainer.java | 3 +-
.../group/MetaoptimizerDistributedStep.java | 13 +-
.../trainers/group/chain/ComputationsChain.java | 4 +-
.../ml/trainers/local/LocalBatchTrainer.java | 178 +++++++++++++++
.../trainers/local/LocalBatchTrainerInput.java | 42 ++++
.../ignite/ml/trainers/local/package-info.java | 22 ++
.../java/org/apache/ignite/ml/util/Utils.java | 22 +-
.../org/apache/ignite/ml/IgniteMLTestSuite.java | 4 +-
.../ignite/ml/nn/MLPGroupTrainerTest.java | 6 +-
.../ignite/ml/nn/MLPLocalTrainerTest.java | 9 +-
.../java/org/apache/ignite/ml/nn/MLPTest.java | 1 +
.../ml/nn/SimpleMLPLocalBatchTrainerInput.java | 1 +
.../ml/nn/performance/MnistDistributed.java | 2 +-
.../ignite/ml/nn/performance/MnistLocal.java | 4 +-
.../ml/optimization/OptimizationTestSuite.java | 33 +++
.../ml/trainers/group/TestGroupTrainer.java | 2 +-
57 files changed, 1384 insertions(+), 1206 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/examples/src/main/ml/org/apache/ignite/examples/ml/nn/MLPGroupTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/nn/MLPGroupTrainerExample.java b/examples/src/main/ml/org/apache/ignite/examples/ml/nn/MLPGroupTrainerExample.java
index 8d4a151..d106fad 100644
--- a/examples/src/main/ml/org/apache/ignite/examples/ml/nn/MLPGroupTrainerExample.java
+++ b/examples/src/main/ml/org/apache/ignite/examples/ml/nn/MLPGroupTrainerExample.java
@@ -35,7 +35,7 @@ import org.apache.ignite.ml.nn.MultilayerPerceptron;
import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
import org.apache.ignite.ml.nn.initializers.RandomInitializer;
import org.apache.ignite.ml.nn.trainers.distributed.MLPGroupUpdateTrainer;
-import org.apache.ignite.ml.nn.updaters.RPropParameterUpdate;
+import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
import org.apache.ignite.ml.structures.LabeledVector;
import org.apache.ignite.thread.IgniteThread;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/examples/src/main/ml/org/apache/ignite/examples/ml/nn/MLPLocalTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/nn/MLPLocalTrainerExample.java b/examples/src/main/ml/org/apache/ignite/examples/ml/nn/MLPLocalTrainerExample.java
index 3f4adc4..b557458 100644
--- a/examples/src/main/ml/org/apache/ignite/examples/ml/nn/MLPLocalTrainerExample.java
+++ b/examples/src/main/ml/org/apache/ignite/examples/ml/nn/MLPLocalTrainerExample.java
@@ -25,13 +25,13 @@ import org.apache.ignite.ml.math.Tracer;
import org.apache.ignite.ml.math.functions.IgniteSupplier;
import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
import org.apache.ignite.ml.nn.Activators;
-import org.apache.ignite.ml.nn.LocalBatchTrainerInput;
-import org.apache.ignite.ml.nn.LossFunctions;
+import org.apache.ignite.ml.trainers.local.LocalBatchTrainerInput;
+import org.apache.ignite.ml.optimization.LossFunctions;
import org.apache.ignite.ml.nn.MultilayerPerceptron;
import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
import org.apache.ignite.ml.nn.initializers.RandomInitializer;
import org.apache.ignite.ml.nn.trainers.local.MLPLocalBatchTrainer;
-import org.apache.ignite.ml.nn.updaters.RPropUpdateCalculator;
+import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
import org.apache.ignite.ml.util.Utils;
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/LocalBatchTrainerInput.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/LocalBatchTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/LocalBatchTrainerInput.java
deleted file mode 100644
index 3a87d02..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/LocalBatchTrainerInput.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn;
-
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-
-/**
- * Interface for classes containing input parameters for LocalBatchTrainer.
- */
-public interface LocalBatchTrainerInput<M extends Model<Matrix, Matrix>> {
- /**
- * Get supplier of next batch in form of matrix of inputs and matrix of outputs.
- *
- * @return Supplier of next batch.
- */
- IgniteSupplier<IgniteBiTuple<Matrix, Matrix>> batchSupplier();
-
- /**
- * Model to train.
- *
- * @return Model to train.
- */
- M mdl();
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/LossFunctions.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/LossFunctions.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/LossFunctions.java
deleted file mode 100644
index dff239c..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/LossFunctions.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn;
-
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-
-/**
- * Class containing popular loss functions.
- */
-public class LossFunctions {
- /**
- * Mean squared error loss function.
- */
- public static IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> MSE = groundTruth ->
- new IgniteDifferentiableVectorToDoubleFunction() {
- /** {@inheritDoc} */
- @Override public Vector differential(Vector pnt) {
- double multiplier = 2.0 / pnt.size();
- return pnt.minus(groundTruth).times(multiplier);
- }
-
- /** {@inheritDoc} */
- @Override public Double apply(Vector vector) {
- return groundTruth.copy().map(vector, (a, b) -> {
- double diff = a - b;
- return diff * diff;
- }).sum() / (vector.size());
- }
- };
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPGroupUpdateTrainerCacheInput.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPGroupUpdateTrainerCacheInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPGroupUpdateTrainerCacheInput.java
index 14db261..05f52e1 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPGroupUpdateTrainerCacheInput.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPGroupUpdateTrainerCacheInput.java
@@ -19,6 +19,7 @@ package org.apache.ignite.ml.nn;
import java.util.ArrayList;
import java.util.List;
+import java.util.Random;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.apache.ignite.Ignite;
@@ -57,6 +58,11 @@ public class MLPGroupUpdateTrainerCacheInput extends AbstractMLPGroupUpdateTrain
private final MultilayerPerceptron mlp;
/**
+ * Random number generator.
+ */
+ private final Random rand;
+
+ /**
* Construct instance of this class with given parameters.
*
* @param arch Architecture of multilayer perceptron.
@@ -64,15 +70,32 @@ public class MLPGroupUpdateTrainerCacheInput extends AbstractMLPGroupUpdateTrain
* @param networksCnt Count of networks to be trained in parallel by {@link MLPGroupUpdateTrainer}.
* @param cache Cache with labeled vectors.
* @param batchSize Size of batch to return on each training iteration.
+ * @param rand RNG.
*/
public MLPGroupUpdateTrainerCacheInput(MLPArchitecture arch, MLPInitializer init,
int networksCnt, IgniteCache<Integer, LabeledVector<Vector, Vector>> cache,
- int batchSize) {
+ int batchSize, Random rand) {
super(networksCnt);
this.batchSize = batchSize;
this.cache = cache;
this.mlp = new MultilayerPerceptron(arch, init);
+ this.rand = rand;
+ }
+
+ /**
+ * Construct instance of this class with given parameters.
+ *
+ * @param arch Architecture of multilayer perceptron.
+ * @param init Initializer of multilayer perceptron.
+ * @param networksCnt Count of networks to be trained in parallel by {@link MLPGroupUpdateTrainer}.
+ * @param cache Cache with labeled vectors.
+ * @param batchSize Size of batch to return on each training iteration.
+ */
+ public MLPGroupUpdateTrainerCacheInput(MLPArchitecture arch, MLPInitializer init,
+ int networksCnt, IgniteCache<Integer, LabeledVector<Vector, Vector>> cache,
+ int batchSize) {
+ this(arch, init, networksCnt, cache, batchSize, new Random());
}
/**
@@ -93,6 +116,7 @@ public class MLPGroupUpdateTrainerCacheInput extends AbstractMLPGroupUpdateTrain
@Override public IgniteSupplier<IgniteBiTuple<Matrix, Matrix>> batchSupplier() {
String cName = cache.getName();
int bs = batchSize;
+ Random r = rand; // IMPL NOTE this is intended to make below lambda more lightweight.
return () -> {
Ignite ignite = Ignition.localIgnite();
@@ -105,7 +129,7 @@ public class MLPGroupUpdateTrainerCacheInput extends AbstractMLPGroupUpdateTrain
int locKeysCnt = keys.size();
- int[] selected = Utils.selectKDistinct(locKeysCnt, Math.min(bs, locKeysCnt));
+ int[] selected = Utils.selectKDistinct(locKeysCnt, Math.min(bs, locKeysCnt), r);
// Get dimensions of vectors in cache. We suppose that every feature vector has
// same dimension d 1 and every label has the same dimension d2.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java
index d55e0e9..7bf238d 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java
@@ -35,7 +35,7 @@ import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
import org.apache.ignite.ml.nn.architecture.TransformationLayerArchitecture;
import org.apache.ignite.ml.nn.initializers.MLPInitializer;
import org.apache.ignite.ml.nn.initializers.RandomInitializer;
-import org.apache.ignite.ml.nn.updaters.SmoothParametrized;
+import org.apache.ignite.ml.optimization.SmoothParametrized;
import static org.apache.ignite.ml.math.util.MatrixUtil.elementWiseTimes;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/AbstractMLPGroupUpdateTrainerInput.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/AbstractMLPGroupUpdateTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/AbstractMLPGroupUpdateTrainerInput.java
index ed65af7..f2d95d5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/AbstractMLPGroupUpdateTrainerInput.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/AbstractMLPGroupUpdateTrainerInput.java
@@ -20,7 +20,7 @@ package org.apache.ignite.ml.nn.trainers.distributed;
import java.util.UUID;
import java.util.stream.Stream;
import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.nn.LocalBatchTrainerInput;
+import org.apache.ignite.ml.trainers.local.LocalBatchTrainerInput;
import org.apache.ignite.ml.nn.MultilayerPerceptron;
import org.apache.ignite.ml.trainers.group.GroupTrainerCacheKey;
import org.apache.ignite.ml.trainers.group.GroupTrainerInput;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainer.java
index 1f98b53..f4647d5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainer.java
@@ -33,11 +33,11 @@ import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFun
import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.ml.math.functions.IgniteSupplier;
import org.apache.ignite.ml.math.util.MatrixUtil;
-import org.apache.ignite.ml.nn.LossFunctions;
+import org.apache.ignite.ml.optimization.LossFunctions;
import org.apache.ignite.ml.nn.MultilayerPerceptron;
-import org.apache.ignite.ml.nn.updaters.ParameterUpdateCalculator;
-import org.apache.ignite.ml.nn.updaters.RPropParameterUpdate;
-import org.apache.ignite.ml.nn.updaters.RPropUpdateCalculator;
+import org.apache.ignite.ml.optimization.updatecalculators.ParameterUpdateCalculator;
+import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
+import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
import org.apache.ignite.ml.trainers.group.GroupTrainerCacheKey;
import org.apache.ignite.ml.trainers.group.MetaoptimizerGroupTrainer;
import org.apache.ignite.ml.trainers.group.ResultAndUpdates;
@@ -227,8 +227,7 @@ public class MLPGroupUpdateTrainer<U extends Serializable> extends
UUID uuid = ctx.trainingUUID();
return () -> {
- MLPGroupUpdateTrainingData<U> data = MLPGroupUpdateTrainerDataCache
- .getOrCreate(Ignition.localIgnite()).get(uuid);
+ MLPGroupUpdateTrainingData<U> data = MLPGroupUpdateTrainerDataCache.getOrCreate(Ignition.localIgnite()).get(uuid);
return new MLPGroupUpdateTrainingContext<>(data, prevUpdate);
};
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainerDataCache.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainerDataCache.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainerDataCache.java
index c237f86..4200321 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainerDataCache.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainerDataCache.java
@@ -55,7 +55,7 @@ public class MLPGroupUpdateTrainerDataCache {
CacheConfiguration<UUID, MLPGroupUpdateTrainingData> cfg = new CacheConfiguration<>();
// Write to primary.
- cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
+ cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
// Atomic transactions only.
cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingData.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingData.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingData.java
index 86074dd..740fac6 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingData.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingData.java
@@ -25,21 +25,36 @@ import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFun
import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.ml.math.functions.IgniteSupplier;
import org.apache.ignite.ml.nn.MultilayerPerceptron;
-import org.apache.ignite.ml.nn.updaters.ParameterUpdateCalculator;
+import org.apache.ignite.ml.optimization.updatecalculators.ParameterUpdateCalculator;
/** Multilayer perceptron group update training data. */
public class MLPGroupUpdateTrainingData<U> {
- /** */
+ /** {@link ParameterUpdateCalculator}. */
private final ParameterUpdateCalculator<MultilayerPerceptron, U> updateCalculator;
- /** */
+
+ /**
+ * Count of steps which should be done by each of parallel trainings before sending it's update for combining with
+ * other parallel trainings updates.
+ */
private final int stepsCnt;
- /** */
+
+ /**
+ * Function used to reduce updates in one training (for example, sum all sequential gradient updates to get one
+ * gradient update).
+ */
private final IgniteFunction<List<U>, U> updateReducer;
- /** */
+
+ /**
+ * Supplier of batches in the form of (inputs, groundTruths).
+ */
private final IgniteSupplier<IgniteBiTuple<Matrix, Matrix>> batchSupplier;
- /** */
+
+ /**
+ * Loss function.
+ */
private final IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
- /** */
+
+ /** Error tolerance. */
private final double tolerance;
/** Construct multilayer perceptron group update training data with all parameters provided. */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingLoopData.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingLoopData.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingLoopData.java
index 0f3d974..2050ee5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingLoopData.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingLoopData.java
@@ -26,29 +26,39 @@ import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFun
import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.ml.math.functions.IgniteSupplier;
import org.apache.ignite.ml.nn.MultilayerPerceptron;
-import org.apache.ignite.ml.nn.updaters.ParameterUpdateCalculator;
+import org.apache.ignite.ml.optimization.updatecalculators.ParameterUpdateCalculator;
import org.apache.ignite.ml.trainers.group.GroupTrainerCacheKey;
/** Multilayer perceptron group update training loop data. */
public class MLPGroupUpdateTrainingLoopData<P> implements Serializable {
- /** */
+ /** {@link ParameterUpdateCalculator}. */
private final ParameterUpdateCalculator<MultilayerPerceptron, P> updateCalculator;
- /** */
+
+ /**
+ * Count of steps which should be done by each of parallel trainings before sending it's update for combining with
+ * other parallel trainings updates.
+ */
private final int stepsCnt;
- /** */
+
+ /** Function used to reduce updates of all steps of given parallel training. */
private final IgniteFunction<List<P>, P> updateReducer;
- /** */
+
+ /** Previous update. */
private final P previousUpdate;
- /** */
+
+ /** Supplier of batches. */
private final IgniteSupplier<IgniteBiTuple<Matrix, Matrix>> batchSupplier;
- /** */
+
+ /** Loss function. */
private final IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
- /** */
+
+ /** Error tolerance. */
private final double tolerance;
- /** */
+ /** Key. */
private final GroupTrainerCacheKey<Void> key;
- /** */
+
+ /** MLP. */
private final MultilayerPerceptron mlp;
/** Create multilayer perceptron group update training loop data. */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPMetaoptimizer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPMetaoptimizer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPMetaoptimizer.java
index 249136b..6e314f1 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPMetaoptimizer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPMetaoptimizer.java
@@ -26,7 +26,7 @@ import org.apache.ignite.ml.trainers.group.Metaoptimizer;
/** Meta-optimizer for multilayer perceptron. */
public class MLPMetaoptimizer<P> implements Metaoptimizer<MLPGroupUpdateTrainerLocalContext,
MLPGroupUpdateTrainingLoopData<P>, P, P, P, ArrayList<P>> {
- /** */
+ /** Function used for reducing updates produced by parallel trainings. */
private final IgniteFunction<List<P>, P> allUpdatesReducer;
/** Construct metaoptimizer. */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/LocalBatchTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/LocalBatchTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/LocalBatchTrainer.java
deleted file mode 100644
index 8579b82..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/LocalBatchTrainer.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.trainers.local;
-
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.math.util.MatrixUtil;
-import org.apache.ignite.ml.nn.LocalBatchTrainerInput;
-import org.apache.ignite.ml.nn.updaters.ParameterUpdateCalculator;
-
-/**
- * Batch trainer. This trainer is not distributed on the cluster, but input can theoretically read data from
- * Ignite cache.
- */
-public class LocalBatchTrainer<M extends Model<Matrix, Matrix>, P>
- implements Trainer<M, LocalBatchTrainerInput<M>> {
- /**
- * Supplier for updater function.
- */
- private final IgniteSupplier<ParameterUpdateCalculator<M, P>> updaterSupplier;
-
- /**
- * Error threshold.
- */
- private final double errorThreshold;
-
- /**
- * Maximal iterations count.
- */
- private final int maxIterations;
-
- /**
- * Loss function.
- */
- private final IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
-
- /**
- * Logger.
- */
- private IgniteLogger log;
-
- /**
- * Construct a trainer.
- *
- * @param loss Loss function.
- * @param updaterSupplier Supplier of updater function.
- * @param errorThreshold Error threshold.
- * @param maxIterations Maximal iterations count.
- */
- public LocalBatchTrainer(IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss,
- IgniteSupplier<ParameterUpdateCalculator<M, P>> updaterSupplier, double errorThreshold, int maxIterations) {
- this.loss = loss;
- this.updaterSupplier = updaterSupplier;
- this.errorThreshold = errorThreshold;
- this.maxIterations = maxIterations;
- }
-
- /** {@inheritDoc} */
- @Override public M train(LocalBatchTrainerInput<M> data) {
- int i = 0;
- M mdl = data.mdl();
- double err;
-
- ParameterUpdateCalculator<? super M, P> updater = updaterSupplier.get();
-
- P updaterParams = updater.init(mdl, loss);
-
- while (i < maxIterations) {
- IgniteBiTuple<Matrix, Matrix> batch = data.batchSupplier().get();
- Matrix input = batch.get1();
- Matrix truth = batch.get2();
-
- updaterParams = updater.calculateNewUpdate(mdl, updaterParams, i, input, truth);
-
- // Update mdl with updater parameters.
- mdl = updater.update(mdl, updaterParams);
-
- Matrix predicted = mdl.apply(input);
-
- int batchSize = input.columnSize();
-
- err = MatrixUtil.zipFoldByColumns(predicted, truth, (predCol, truthCol) ->
- loss.apply(truthCol).apply(predCol)).sum() / batchSize;
-
- debug("Error: " + err);
-
- if (err < errorThreshold)
- break;
-
- i++;
- }
-
- return mdl;
- }
-
- /**
- * Construct new trainer with the same parameters as this trainer, but with new loss.
- *
- * @param loss New loss function.
- * @return new trainer with the same parameters as this trainer, but with new loss.
- */
- public LocalBatchTrainer withLoss(IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss) {
- return new LocalBatchTrainer<>(loss, updaterSupplier, errorThreshold, maxIterations);
- }
-
- /**
- * Construct new trainer with the same parameters as this trainer, but with new updater supplier.
- *
- * @param updaterSupplier New updater supplier.
- * @return new trainer with the same parameters as this trainer, but with new updater supplier.
- */
- public LocalBatchTrainer withUpdater(IgniteSupplier<ParameterUpdateCalculator<M, P>> updaterSupplier) {
- return new LocalBatchTrainer<>(loss, updaterSupplier, errorThreshold, maxIterations);
- }
-
- /**
- * Construct new trainer with the same parameters as this trainer, but with new error threshold.
- *
- * @param errorThreshold New error threshold.
- * @return new trainer with the same parameters as this trainer, but with new error threshold.
- */
- public LocalBatchTrainer withErrorThreshold(double errorThreshold) {
- return new LocalBatchTrainer<>(loss, updaterSupplier, errorThreshold, maxIterations);
- }
-
- /**
- * Construct new trainer with the same parameters as this trainer, but with new maximal iterations count.
- *
- * @param maxIterations New maximal iterations count.
- * @return new trainer with the same parameters as this trainer, but with new maximal iterations count.
- */
- public LocalBatchTrainer withMaxIterations(int maxIterations) {
- return new LocalBatchTrainer<>(loss, updaterSupplier, errorThreshold, maxIterations);
- }
-
- /**
- * Set logger.
- *
- * @param log Logger.
- * @return This object.
- */
- public LocalBatchTrainer setLogger(IgniteLogger log) {
- this.log = log;
-
- return this;
- }
-
- /**
- * Output debug message.
- *
- * @param msg Message.
- */
- private void debug(String msg) {
- if (log != null && log.isDebugEnabled())
- log.debug(msg);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/MLPLocalBatchTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/MLPLocalBatchTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/MLPLocalBatchTrainer.java
index 0c92395..059d15a 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/MLPLocalBatchTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/MLPLocalBatchTrainer.java
@@ -21,11 +21,12 @@ import org.apache.ignite.ml.math.Vector;
import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.nn.LossFunctions;
+import org.apache.ignite.ml.optimization.LossFunctions;
import org.apache.ignite.ml.nn.MultilayerPerceptron;
-import org.apache.ignite.ml.nn.updaters.ParameterUpdateCalculator;
-import org.apache.ignite.ml.nn.updaters.RPropParameterUpdate;
-import org.apache.ignite.ml.nn.updaters.RPropUpdateCalculator;
+import org.apache.ignite.ml.optimization.updatecalculators.ParameterUpdateCalculator;
+import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
+import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
+import org.apache.ignite.ml.trainers.local.LocalBatchTrainer;
/**
* Local batch trainer for MLP.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/BaseSmoothParametrized.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/BaseSmoothParametrized.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/BaseSmoothParametrized.java
deleted file mode 100644
index 8e2f0df..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/BaseSmoothParametrized.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-
-/**
- * Interface for models which are smooth functions of their parameters.
- */
-interface BaseSmoothParametrized<M extends BaseSmoothParametrized<M> & Model<Matrix, Matrix>> {
- /**
- * Compose function in the following way: feed output of this model as input to second argument to loss function.
- * After that we have a function g of three arguments: input, ground truth, parameters.
- * If we consider function
- * h(w) = 1 / M sum_{i=1}^{M} g(w, input_i, groundTruth_i),
- * where M is number of entries in batch, we get function of one argument: parameters vector w.
- * This function is being differentiated.
- *
- * @param loss Loss function.
- * @param inputsBatch Batch of inputs.
- * @param truthBatch Batch of ground truths.
- * @return Gradient of h at current point in parameters space.
- */
- Vector differentiateByParameters(IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss,
- Matrix inputsBatch, Matrix truthBatch);
-
- /**
- * Get parameters vector.
- *
- * @return Parameters vector.
- */
- Vector parameters();
-
- /**
- * Set parameters.
- *
- * @param vector Parameters vector.
- */
- M setParameters(Vector vector);
-
- /**
- * Get count of parameters of this model.
- *
- * @return Count of parameters of this model.
- */
- int parametersCount();
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovParameterUpdate.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovParameterUpdate.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovParameterUpdate.java
deleted file mode 100644
index 8671285..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovParameterUpdate.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Objects;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-
-/**
- * Data needed for Nesterov parameters updater.
- */
-public class NesterovParameterUpdate implements Serializable {
- /**
- * Previous step weights updates.
- */
- protected Vector prevIterationUpdates;
-
- /**
- * Construct NesterovParameterUpdate.
- *
- * @param paramsCnt Count of parameters on which updateCache happens.
- */
- public NesterovParameterUpdate(int paramsCnt) {
- prevIterationUpdates = new DenseLocalOnHeapVector(paramsCnt).assign(0);
- }
-
- /**
- * Construct NesterovParameterUpdate.
- *
- * @param prevIterationUpdates Previous iteration updates.
- */
- public NesterovParameterUpdate(Vector prevIterationUpdates) {
- this.prevIterationUpdates = prevIterationUpdates;
- }
-
- /**
- * Set previous step parameters updates.
- *
- * @param updates Parameters updates.
- * @return This object with updated parameters updates.
- */
- public NesterovParameterUpdate setPreviousUpdates(Vector updates) {
- prevIterationUpdates = updates;
- return this;
- }
-
- /**
- * Get previous step parameters updates.
- *
- * @return Previous step parameters updates.
- */
- public Vector prevIterationUpdates() {
- return prevIterationUpdates;
- }
-
- /**
- * Get sum of parameters updates.
- *
- * @param parameters Parameters to sum.
- * @return Sum of parameters updates.
- */
- public static NesterovParameterUpdate sum(List<NesterovParameterUpdate> parameters) {
- return parameters.stream().filter(Objects::nonNull).map(NesterovParameterUpdate::prevIterationUpdates)
- .reduce(Vector::plus).map(NesterovParameterUpdate::new).orElse(null);
- }
-
- /**
- * Get average of parameters updates.
- *
- * @param parameters Parameters to average.
- * @return Average of parameters updates.
- */
- public static NesterovParameterUpdate avg(List<NesterovParameterUpdate> parameters) {
- NesterovParameterUpdate sum = sum(parameters);
- return sum != null ? sum.setPreviousUpdates(sum.prevIterationUpdates().divide(parameters.size())) : null;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdateCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdateCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdateCalculator.java
deleted file mode 100644
index 5caddd4..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/NesterovUpdateCalculator.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-
-/**
- * Class encapsulating Nesterov algorithm for MLP parameters updateCache.
- */
-public class NesterovUpdateCalculator<M extends SmoothParametrized>
- implements ParameterUpdateCalculator<M, NesterovParameterUpdate> {
- /**
- * Learning rate.
- */
- private final double learningRate;
-
- /**
- * Loss function.
- */
- private IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
-
- /**
- * Momentum constant.
- */
- protected double momentum;
-
- /**
- * Construct NesterovUpdateCalculator.
- *
- * @param momentum Momentum constant.
- */
- public NesterovUpdateCalculator(double learningRate, double momentum) {
- this.learningRate = learningRate;
- this.momentum = momentum;
- }
-
- /** {@inheritDoc} */
- @Override public NesterovParameterUpdate calculateNewUpdate(SmoothParametrized mdl,
- NesterovParameterUpdate updaterParameters, int iteration, Matrix inputs, Matrix groundTruth) {
- // TODO:IGNITE-7350 create new updateCache object here instead of in-place change.
-
- if (iteration > 0) {
- Vector curParams = mdl.parameters();
- mdl.setParameters(curParams.minus(updaterParameters.prevIterationUpdates().times(momentum)));
- }
-
- Vector gradient = mdl.differentiateByParameters(loss, inputs, groundTruth);
- updaterParameters.setPreviousUpdates(updaterParameters.prevIterationUpdates()
- .plus(gradient.times(learningRate)));
-
- return updaterParameters;
- }
-
- /** {@inheritDoc} */
- @Override public NesterovParameterUpdate init(M mdl,
- IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss) {
- this.loss = loss;
-
- return new NesterovParameterUpdate(mdl.parametersCount());
- }
-
- /** {@inheritDoc} */
- @Override public <M1 extends M> M1 update(M1 obj, NesterovParameterUpdate update) {
- Vector parameters = obj.parameters();
- return (M1)obj.setParameters(parameters.minus(update.prevIterationUpdates()));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/ParameterUpdateCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/ParameterUpdateCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/ParameterUpdateCalculator.java
deleted file mode 100644
index 77e3763..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/ParameterUpdateCalculator.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-
-/**
- * Interface for classes encapsulating parameters updateCache logic.
- *
- * @param <M> Type of model to be updated.
- * @param <P> Type of parameters needed for this updater.
- */
-public interface ParameterUpdateCalculator<M, P> {
- /**
- * Initializes the updater.
- *
- * @param mdl Model to be trained.
- * @param loss Loss function.
- */
- P init(M mdl, IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss);
-
- /**
- * Calculate new updateCache.
- *
- * @param mdl Model to be updated.
- * @param updaterParameters Updater parameters to updateCache.
- * @param iteration Current trainer iteration.
- * @param inputs Inputs.
- * @param groundTruth True values.
- * @return Updated parameters.
- */
- P calculateNewUpdate(M mdl, P updaterParameters, int iteration, Matrix inputs, Matrix groundTruth);
-
- /**
- * Update given obj with this parameters.
- *
- * @param obj Object to be updated.
- */
- <M1 extends M> M1 update(M1 obj, P update);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropParameterUpdate.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropParameterUpdate.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropParameterUpdate.java
deleted file mode 100644
index e2fa4d5..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropParameterUpdate.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.VectorUtils;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-
-/**
- * Data needed for RProp updater.
- * <p>
- * See <a href="https://paginas.fe.up.pt/~ee02162/dissertacao/RPROP%20paper.pdf">RProp</a>.</p>
- */
-public class RPropParameterUpdate implements Serializable {
- /**
- * Previous iteration parameters updates. In original paper they are labeled with "delta w".
- */
- protected Vector prevIterationUpdates;
-
- /**
- * Previous iteration model partial derivatives by parameters.
- */
- protected Vector prevIterationGradient;
- /**
- * Previous iteration parameters deltas. In original paper they are labeled with "delta".
- */
- protected Vector deltas;
-
- /**
- * Updates mask (values by which updateCache is multiplied).
- */
- protected Vector updatesMask;
-
- /**
- * Construct RPropParameterUpdate.
- *
- * @param paramsCnt Parameters count.
- * @param initUpdate Initial updateCache (in original work labeled as "delta_0").
- */
- RPropParameterUpdate(int paramsCnt, double initUpdate) {
- prevIterationUpdates = new DenseLocalOnHeapVector(paramsCnt);
- prevIterationGradient = new DenseLocalOnHeapVector(paramsCnt);
- deltas = new DenseLocalOnHeapVector(paramsCnt).assign(initUpdate);
- updatesMask = new DenseLocalOnHeapVector(paramsCnt);
- }
-
- /**
- * Construct instance of this class by given parameters.
- *
- * @param prevIterationUpdates Previous iteration parameters updates.
- * @param prevIterationGradient Previous iteration model partial derivatives by parameters.
- * @param deltas Previous iteration parameters deltas.
- * @param updatesMask Updates mask.
- */
- public RPropParameterUpdate(Vector prevIterationUpdates, Vector prevIterationGradient,
- Vector deltas, Vector updatesMask) {
- this.prevIterationUpdates = prevIterationUpdates;
- this.prevIterationGradient = prevIterationGradient;
- this.deltas = deltas;
- this.updatesMask = updatesMask;
- }
-
- /**
- * Get bias deltas.
- *
- * @return Bias deltas.
- */
- Vector deltas() {
- return deltas;
- }
-
- /**
- * Get previous iteration biases updates. In original paper they are labeled with "delta w".
- *
- * @return Biases updates.
- */
- Vector prevIterationUpdates() {
- return prevIterationUpdates;
- }
-
- /**
- * Set previous iteration parameters updates. In original paper they are labeled with "delta w".
- *
- * @param updates New parameters updates value.
- * @return This object.
- */
- private RPropParameterUpdate setPrevIterationUpdates(Vector updates) {
- prevIterationUpdates = updates;
-
- return this;
- }
-
- /**
- * Get previous iteration loss function partial derivatives by parameters.
- *
- * @return Previous iteration loss function partial derivatives by parameters.
- */
- Vector prevIterationGradient() {
- return prevIterationGradient;
- }
-
- /**
- * Set previous iteration loss function partial derivatives by parameters.
- *
- * @return This object.
- */
- private RPropParameterUpdate setPrevIterationGradient(Vector gradient) {
- prevIterationGradient = gradient;
- return this;
- }
-
- /**
- * Get updates mask (values by which updateCache is multiplied).
- *
- * @return Updates mask (values by which updateCache is multiplied).
- */
- public Vector updatesMask() {
- return updatesMask;
- }
-
- /**
- * Set updates mask (values by which updateCache is multiplied).
- *
- * @param updatesMask New updatesMask.
- * @return This object.
- */
- public RPropParameterUpdate setUpdatesMask(Vector updatesMask) {
- this.updatesMask = updatesMask;
-
- return this;
- }
-
- /**
- * Set previous iteration deltas.
- *
- * @param deltas New deltas.
- * @return This object.
- */
- public RPropParameterUpdate setDeltas(Vector deltas) {
- this.deltas = deltas;
-
- return this;
- }
-
- /**
- * Sums updates during one training.
- *
- * @param updates Updates.
- * @return Sum of updates during one training.
- */
- public static RPropParameterUpdate sumLocal(List<RPropParameterUpdate> updates) {
- List<RPropParameterUpdate> nonNullUpdates = updates.stream().filter(Objects::nonNull)
- .collect(Collectors.toList());
-
- if (nonNullUpdates.isEmpty())
- return null;
-
- Vector newDeltas = nonNullUpdates.get(nonNullUpdates.size() - 1).deltas();
- Vector newGradient = nonNullUpdates.get(nonNullUpdates.size() - 1).prevIterationGradient();
- Vector totalUpdate = nonNullUpdates.stream().map(pu -> VectorUtils.elementWiseTimes(pu.updatesMask().copy(),
- pu.prevIterationUpdates())).reduce(Vector::plus).orElse(null);
-
- return new RPropParameterUpdate(totalUpdate, newGradient, newDeltas,
- new DenseLocalOnHeapVector(newDeltas.size()).assign(1.0));
- }
-
- /**
- * Sums updates returned by different trainings.
- *
- * @param updates Updates.
- * @return Sum of updates during returned by different trainings.
- */
- public static RPropParameterUpdate sum(List<RPropParameterUpdate> updates) {
- Vector totalUpdate = updates.stream().filter(Objects::nonNull)
- .map(pu -> VectorUtils.elementWiseTimes(pu.updatesMask().copy(), pu.prevIterationUpdates()))
- .reduce(Vector::plus).orElse(null);
- Vector totalDelta = updates.stream().filter(Objects::nonNull)
- .map(RPropParameterUpdate::deltas).reduce(Vector::plus).orElse(null);
- Vector totalGradient = updates.stream().filter(Objects::nonNull)
- .map(RPropParameterUpdate::prevIterationGradient).reduce(Vector::plus).orElse(null);
-
- if (totalUpdate != null)
- return new RPropParameterUpdate(totalUpdate, totalGradient, totalDelta,
- new DenseLocalOnHeapVector(Objects.requireNonNull(totalDelta).size()).assign(1.0));
-
- return null;
- }
-
- /**
- * Averages updates returned by different trainings.
- *
- * @param updates Updates.
- * @return Averages of updates during returned by different trainings.
- */
- public static RPropParameterUpdate avg(List<RPropParameterUpdate> updates) {
- List<RPropParameterUpdate> nonNullUpdates = updates.stream()
- .filter(Objects::nonNull).collect(Collectors.toList());
- int size = nonNullUpdates.size();
-
- RPropParameterUpdate sum = sum(updates);
- if (sum != null)
- return sum.
- setPrevIterationGradient(sum.prevIterationGradient().divide(size)).
- setPrevIterationUpdates(sum.prevIterationUpdates().divide(size)).
- setDeltas(sum.deltas().divide(size));
-
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdateCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdateCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdateCalculator.java
deleted file mode 100644
index 99f39c9..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/RPropUpdateCalculator.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.VectorUtils;
-import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.util.MatrixUtil;
-
-/**
- * Class encapsulating RProp algorithm.
- * <p>
- * See <a href="https://paginas.fe.up.pt/~ee02162/dissertacao/RPROP%20paper.pdf">RProp</a>.</p>
- */
-public class RPropUpdateCalculator<M extends SmoothParametrized> implements ParameterUpdateCalculator<M, RPropParameterUpdate> {
- /**
- * Default initial update.
- */
- private static double DFLT_INIT_UPDATE = 0.1;
-
- /**
- * Default acceleration rate.
- */
- private static double DFLT_ACCELERATION_RATE = 1.2;
-
- /**
- * Default deacceleration rate.
- */
- private static double DFLT_DEACCELERATION_RATE = 0.5;
-
- /**
- * Initial update.
- */
- private final double initUpdate;
-
- /**
- * Acceleration rate.
- */
- private final double accelerationRate;
-
- /**
- * Deacceleration rate.
- */
- private final double deaccelerationRate;
-
- /**
- * Maximal value for update.
- */
- private final static double UPDATE_MAX = 50.0;
-
- /**
- * Minimal value for update.
- */
- private final static double UPDATE_MIN = 1E-6;
-
- /**
- * Loss function.
- */
- protected IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
-
- /**
- * Construct RPropUpdateCalculator.
- *
- * @param initUpdate Initial update.
- * @param accelerationRate Acceleration rate.
- * @param deaccelerationRate Deacceleration rate.
- */
- public RPropUpdateCalculator(double initUpdate, double accelerationRate, double deaccelerationRate) {
- this.initUpdate = initUpdate;
- this.accelerationRate = accelerationRate;
- this.deaccelerationRate = deaccelerationRate;
- }
-
- /**
- * Construct RPropUpdateCalculator with default parameters.
- */
- public RPropUpdateCalculator() {
- this(DFLT_INIT_UPDATE, DFLT_ACCELERATION_RATE, DFLT_DEACCELERATION_RATE);
- }
-
- /** {@inheritDoc} */
- @Override public RPropParameterUpdate calculateNewUpdate(SmoothParametrized mdl, RPropParameterUpdate updaterParams,
- int iteration, Matrix inputs, Matrix groundTruth) {
- Vector gradient = mdl.differentiateByParameters(loss, inputs, groundTruth);
- Vector prevGradient = updaterParams.prevIterationGradient();
- Vector derSigns;
-
- if (prevGradient != null)
- derSigns = VectorUtils.zipWith(prevGradient, gradient, (x, y) -> Math.signum(x * y));
- else
- derSigns = gradient.like(gradient.size()).assign(1.0);
-
- Vector newDeltas = updaterParams.deltas().copy().map(derSigns, (prevDelta, sign) -> {
- if (sign > 0)
- return Math.min(prevDelta * accelerationRate, UPDATE_MAX);
- else if (sign < 0)
- return Math.max(prevDelta * deaccelerationRate, UPDATE_MIN);
- else
- return prevDelta;
- });
-
- Vector newPrevIterationUpdates = MatrixUtil.zipWith(gradient, updaterParams.deltas(), (der, delta, i) -> {
- if (derSigns.getX(i) >= 0)
- return -Math.signum(der) * delta;
-
- return updaterParams.prevIterationUpdates().getX(i);
- });
-
- Vector updatesMask = MatrixUtil.zipWith(derSigns, updaterParams.prevIterationUpdates(), (sign, upd, i) -> {
- if (sign < 0)
- gradient.setX(i, 0.0);
-
- if (sign >= 0)
- return 1.0;
- else
- return -1.0;
- });
-
- return new RPropParameterUpdate(newPrevIterationUpdates, gradient.copy(), newDeltas, updatesMask);
- }
-
- /** {@inheritDoc} */
- @Override public RPropParameterUpdate init(M mdl,
- IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss) {
- this.loss = loss;
- return new RPropParameterUpdate(mdl.parametersCount(), initUpdate);
- }
-
- /** {@inheritDoc} */
- @Override public <M1 extends M> M1 update(M1 obj, RPropParameterUpdate update) {
- Vector updatesToAdd = VectorUtils.elementWiseTimes(update.updatesMask().copy(), update.prevIterationUpdates());
- return (M1)obj.setParameters(obj.parameters().plus(updatesToAdd));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDParameter.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDParameter.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDParameter.java
deleted file mode 100644
index 7159621..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDParameter.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import java.io.Serializable;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-
-/**
- * Parameters for {@link SimpleGDUpdateCalculator}.
- */
-public class SimpleGDParameter implements Serializable {
- /**
- * Gradient.
- */
- private Vector gradient;
-
- /**
- * Learning rate.
- */
- private double learningRate;
-
- /**
- * Construct instance of this class.
- *
- * @param paramsCnt Count of parameters.
- * @param learningRate Learning rate.
- */
- public SimpleGDParameter(int paramsCnt, double learningRate) {
- gradient = new DenseLocalOnHeapVector(paramsCnt);
- this.learningRate = learningRate;
- }
-
- /**
- * Construct instance of this class.
- *
- * @param gradient Gradient.
- * @param learningRate Learning rate.
- */
- public SimpleGDParameter(Vector gradient, double learningRate) {
- this.gradient = gradient;
- this.learningRate = learningRate;
- }
-
- /**
- * Get gradient.
- *
- * @return Get gradient.
- */
- public Vector gradient() {
- return gradient;
- }
-
- /**
- * Get learning rate.
- *
- * @return learning rate.
- */
- public double learningRate() {
- return learningRate;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDUpdateCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDUpdateCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDUpdateCalculator.java
deleted file mode 100644
index d2197d9..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SimpleGDUpdateCalculator.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-
-/**
- * Simple gradient descent parameters updater.
- */
-public class SimpleGDUpdateCalculator<M extends SmoothParametrized> implements ParameterUpdateCalculator<M, SimpleGDParameter> {
- /**
- * Learning rate.
- */
- private double learningRate;
-
- /**
- * Loss function.
- */
- protected IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
-
- /**
- * Construct SimpleGDUpdateCalculator.
- *
- * @param learningRate Learning rate.
- */
- public SimpleGDUpdateCalculator(double learningRate) {
- this.learningRate = learningRate;
- }
-
- /** {@inheritDoc} */
- @Override public SimpleGDParameter init(M mdl,
- IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss) {
- this.loss = loss;
- return new SimpleGDParameter(mdl.parametersCount(), learningRate);
- }
-
- /** {@inheritDoc} */
- @Override public SimpleGDParameter calculateNewUpdate(SmoothParametrized mlp, SimpleGDParameter updaterParameters,
- int iteration, Matrix inputs, Matrix groundTruth) {
- return new SimpleGDParameter(mlp.differentiateByParameters(loss, inputs, groundTruth), learningRate);
- }
-
- /** {@inheritDoc} */
- @Override public <M1 extends M> M1 update(M1 obj, SimpleGDParameter update) {
- Vector params = obj.parameters();
- return (M1)obj.setParameters(params.minus(update.gradient().times(learningRate)));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SmoothParametrized.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SmoothParametrized.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SmoothParametrized.java
deleted file mode 100644
index 1534a6d..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/SmoothParametrized.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.nn.updaters;
-
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.math.Matrix;
-
-/**
- * Interface for models which are smooth functions of their parameters.
- */
-public interface SmoothParametrized<M extends SmoothParametrized<M>> extends BaseSmoothParametrized<M>, Model<Matrix, Matrix> {
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/package-info.java
deleted file mode 100644
index 13bc3c8..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/updaters/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains parameters updaters.
- */
-package org.apache.ignite.ml.nn.updaters;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/optimization/BarzilaiBorweinUpdater.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/BarzilaiBorweinUpdater.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/BarzilaiBorweinUpdater.java
index 2190d86..9b98ad8 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/BarzilaiBorweinUpdater.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/BarzilaiBorweinUpdater.java
@@ -34,8 +34,11 @@ public class BarzilaiBorweinUpdater implements Updater {
/**
* {@inheritDoc}
*/
- @Override public Vector compute(Vector oldWeights, Vector oldGradient, Vector weights, Vector gradient, int iteration) {
- double learningRate = computeLearningRate(oldWeights != null ? oldWeights.copy() : null, oldGradient != null ? oldGradient.copy() : null, weights.copy(), gradient.copy());
+ @Override public Vector compute(Vector oldWeights, Vector oldGradient, Vector weights, Vector gradient,
+ int iteration) {
+ double learningRate = computeLearningRate(oldWeights != null ? oldWeights.copy() : null,
+ oldGradient != null ? oldGradient.copy() : null, weights.copy(), gradient.copy());
+
return weights.copy().minus(gradient.copy().times(learningRate));
}
@@ -45,6 +48,7 @@ public class BarzilaiBorweinUpdater implements Updater {
return INITIAL_LEARNING_RATE;
else {
Vector gradientDiff = gradient.minus(oldGradient);
+
return weights.minus(oldWeights).dot(gradientDiff) / Math.pow(gradientDiff.kNorm(2.0), 2.0);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/optimization/BaseParametrized.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/BaseParametrized.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/BaseParametrized.java
new file mode 100644
index 0000000..c5b2423
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/BaseParametrized.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.optimization;
+
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.util.Utils;
+
+/**
+ * Base interface for parametrized models.
+ *
+ * @param <M> Model class.
+ */
+interface BaseParametrized<M extends BaseParametrized<M>> {
+ /**
+ * Get parameters vector.
+ *
+ * @return Parameters vector.
+ */
+ Vector parameters();
+
+ /**
+ * Set parameters.
+ *
+ * @param vector Parameters vector.
+ */
+ M setParameters(Vector vector);
+
+ /**
+ * Return new model with given parameters vector.
+ *
+ * @param vector Parameters vector.
+ */
+ default M withParameters(Vector vector) {
+ return Utils.copy(this).setParameters(vector);
+ }
+
+ /**
+ * Get count of parameters of this model.
+ *
+ * @return Count of parameters of this model.
+ */
+ int parametersCount();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientDescent.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientDescent.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientDescent.java
index f02bcb3..15ed914 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientDescent.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientDescent.java
@@ -141,6 +141,7 @@ public class GradientDescent {
cnt++;
}
}
+
return resGradient.divide(cnt);
},
weights);
@@ -187,7 +188,7 @@ public class GradientDescent {
/** Makes carrying of the gradient function and fixes data matrix. */
private IgniteFunction<Vector, Vector> getLossGradientFunction(Matrix data) {
if (data instanceof SparseDistributedMatrix) {
- SparseDistributedMatrix distributedMatrix = (SparseDistributedMatrix) data;
+ SparseDistributedMatrix distributedMatrix = (SparseDistributedMatrix)data;
if (distributedMatrix.getStorage().storageMode() == StorageConstants.ROW_STORAGE_MODE)
return weights -> calculateDistributedGradient(distributedMatrix, weights);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientFunction.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientFunction.java
index 7dc6674..a6a1e71 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientFunction.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientFunction.java
@@ -27,5 +27,5 @@ import org.apache.ignite.ml.math.Vector;
@FunctionalInterface
public interface GradientFunction extends Serializable {
/** */
- Vector compute(Matrix inputs, Vector groundTruth, Vector point);
+ Vector compute(Matrix inputs, Vector groundTruth, Vector pnt);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/optimization/LossFunctions.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/LossFunctions.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/LossFunctions.java
new file mode 100644
index 0000000..13fcb60
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/LossFunctions.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.optimization;
+
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+
+/**
+ * Class containing popular loss functions.
+ */
+public class LossFunctions {
+ /**
+ * Mean squared error loss function.
+ */
+ public static IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> MSE = groundTruth ->
+ new IgniteDifferentiableVectorToDoubleFunction() {
+ /** {@inheritDoc} */
+ @Override public Vector differential(Vector pnt) {
+ double multiplier = 2.0 / pnt.size();
+ return pnt.minus(groundTruth).times(multiplier);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Double apply(Vector vector) {
+ return groundTruth.copy().map(vector, (a, b) -> {
+ double diff = a - b;
+ return diff * diff;
+ }).sum() / (vector.size());
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/optimization/Parametrized.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/Parametrized.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/Parametrized.java
new file mode 100644
index 0000000..d64d9c3
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/Parametrized.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.optimization;
+
+/**
+ * Interface for parametrized models.
+ *
+ * @param <M> Type of model.
+ */
+public interface Parametrized<M extends Parametrized<M>> extends BaseParametrized<M> {
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/optimization/SimpleUpdater.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/SimpleUpdater.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/SimpleUpdater.java
index 0f6d520..decbb86 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/SimpleUpdater.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/SimpleUpdater.java
@@ -39,7 +39,8 @@ public class SimpleUpdater implements Updater {
/**
* {@inheritDoc}
*/
- @Override public Vector compute(Vector oldWeights, Vector oldGradient, Vector weights, Vector gradient, int iteration) {
+ @Override public Vector compute(Vector oldWeights, Vector oldGradient, Vector weights, Vector gradient,
+ int iteration) {
return weights.minus(gradient.times(learningRate));
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6eccf230/modules/ml/src/main/java/org/apache/ignite/ml/optimization/SmoothParametrized.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/SmoothParametrized.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/SmoothParametrized.java
new file mode 100644
index 0000000..862a78f
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/SmoothParametrized.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.optimization;
+
+import org.apache.ignite.ml.Model;
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+
+/**
+ * Interface for models which are smooth functions of their parameters.
+ */
+public interface SmoothParametrized<M extends Parametrized<M>> extends Parametrized<M>, Model<Matrix, Matrix> {
+ /**
+ * Compose function in the following way: feed output of this model as input to second argument to loss function.
+ * After that we have a function g of three arguments: input, ground truth, parameters.
+ * If we consider function
+ * h(w) = 1 / M sum_{i=1}^{M} g(w, input_i, groundTruth_i),
+ * where M is number of entries in batch, we get function of one argument: parameters vector w.
+ * This function is being differentiated.
+ *
+ * @param loss Loss function.
+ * @param inputsBatch Batch of inputs.
+ * @param truthBatch Batch of ground truths.
+ * @return Gradient of h at current point in parameters space.
+ */
+ Vector differentiateByParameters(IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss,
+ Matrix inputsBatch, Matrix truthBatch);
+}