You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2017/12/25 11:46:48 UTC
[16/20] ignite git commit: IGNITE-7174: Local MLP
IGNITE-7174: Local MLP
this closes #3246
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e4f19215
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e4f19215
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e4f19215
Branch: refs/heads/ignite-zk
Commit: e4f19215de8d20311e16a6b12d0ef68d711b3462
Parents: 661ada6
Author: artemmalykh <am...@gridgain.com>
Authored: Fri Dec 22 18:07:44 2017 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Fri Dec 22 18:07:44 2017 +0300
----------------------------------------------------------------------
.../KNNClassificationExample.java | 2 +-
.../ml/knn/regression/KNNRegressionExample.java | 2 +-
.../DistributedRegressionModelExample.java | 2 +-
.../main/java/org/apache/ignite/ml/Model.java | 10 +-
.../ignite/ml/clustering/FuzzyCMeansModel.java | 2 +-
.../ignite/ml/clustering/KMeansModel.java | 2 +-
.../apache/ignite/ml/estimators/Estimators.java | 4 +-
.../apache/ignite/ml/knn/models/KNNModel.java | 2 +-
.../regression/KNNMultipleLinearRegression.java | 2 +-
.../org/apache/ignite/ml/math/VectorUtils.java | 96 +++
...iteDifferentiableDoubleToDoubleFunction.java | 31 +
...iteDifferentiableVectorToDoubleFunction.java | 33 ++
.../ml/math/functions/IgniteTriConsumer.java | 40 ++
.../ml/math/impls/matrix/AbstractMatrix.java | 5 +
.../apache/ignite/ml/math/util/MatrixUtil.java | 88 +++
.../org/apache/ignite/ml/nn/Activators.java | 61 ++
.../ignite/ml/nn/LocalBatchTrainerInput.java | 41 ++
.../org/apache/ignite/ml/nn/LossFunctions.java | 47 ++
.../java/org/apache/ignite/ml/nn/MLPLayer.java | 47 ++
.../java/org/apache/ignite/ml/nn/MLPState.java | 73 +++
.../ignite/ml/nn/MultilayerPerceptron.java | 565 ++++++++++++++++++
.../ignite/ml/nn/ReplicatedVectorMatrix.java | 583 +++++++++++++++++++
.../ml/nn/architecture/LayerArchitecture.java | 45 ++
.../ml/nn/architecture/MLPArchitecture.java | 147 +++++
.../TransformationLayerArchitecture.java | 68 +++
.../ignite/ml/nn/architecture/package-info.java | 22 +
.../ml/nn/initializers/MLPInitializer.java | 40 ++
.../ml/nn/initializers/RandomInitializer.java | 51 ++
.../ignite/ml/nn/initializers/package-info.java | 22 +
.../org/apache/ignite/ml/nn/package-info.java | 22 +
.../ml/nn/trainers/local/LocalBatchTrainer.java | 180 ++++++
.../nn/trainers/local/MLPLocalBatchTrainer.java | 78 +++
.../ml/nn/trainers/local/package-info.java | 22 +
.../ignite/ml/nn/trainers/package-info.java | 22 +
.../ml/nn/updaters/BaseSmoothParametrized.java | 64 ++
.../ignite/ml/nn/updaters/NesterovUpdater.java | 76 +++
.../ml/nn/updaters/NesterovUpdaterParams.java | 67 +++
.../ignite/ml/nn/updaters/ParameterUpdater.java | 51 ++
.../ignite/ml/nn/updaters/RPropUpdater.java | 148 +++++
.../ml/nn/updaters/RPropUpdaterParams.java | 134 +++++
.../ignite/ml/nn/updaters/SimpleGDParams.java | 65 +++
.../ignite/ml/nn/updaters/SimpleGDUpdater.java | 60 ++
.../ml/nn/updaters/SmoothParametrized.java | 24 +
.../ignite/ml/nn/updaters/UpdaterParams.java | 32 +
.../ignite/ml/nn/updaters/package-info.java | 22 +
.../OLSMultipleLinearRegressionModel.java | 2 +-
.../ml/trees/models/DecisionTreeModel.java | 2 +-
.../java/org/apache/ignite/ml/util/Utils.java | 27 +
.../org/apache/ignite/ml/IgniteMLTestSuite.java | 4 +-
.../ignite/ml/knn/KNNClassificationTest.java | 14 +-
.../ml/knn/KNNMultipleLinearRegressionTest.java | 16 +-
.../ignite/ml/nn/MLPConstInitializer.java | 67 +++
.../ignite/ml/nn/MLPLocalTrainerTest.java | 97 +++
.../java/org/apache/ignite/ml/nn/MLPTest.java | 207 +++++++
.../org/apache/ignite/ml/nn/MLPTestSuite.java | 33 ++
.../ml/nn/SimpleMLPLocalBatchTrainerInput.java | 95 +++
.../apache/ignite/ml/nn/performance/Mnist.java | 140 +++++
.../OLSMultipleLinearRegressionModelTest.java | 2 +-
.../ml/trees/ColumnDecisionTreeTrainerTest.java | 4 +-
.../ColumnDecisionTreeTrainerBenchmark.java | 4 +-
.../trees/columntrees.manualrun.properties | 8 +-
.../yardstick/ml/trees/SplitDataGenerator.java | 2 +-
62 files changed, 3880 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/KNNClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/KNNClassificationExample.java b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/KNNClassificationExample.java
index a92e9af..fb7eebd 100644
--- a/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/KNNClassificationExample.java
+++ b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/KNNClassificationExample.java
@@ -89,7 +89,7 @@ public class KNNClassificationExample {
// Save predicted classes to test dataset
for (int i = 0; i < test.rowSize(); i++) {
- double predictedCls = knnMdl.predict(test.getRow(i).features());
+ double predictedCls = knnMdl.apply(test.getRow(i).features());
test.setLabel(i, predictedCls);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/KNNRegressionExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/KNNRegressionExample.java b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/KNNRegressionExample.java
index f4a9e1c..6ed0dd6 100644
--- a/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/KNNRegressionExample.java
+++ b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/KNNRegressionExample.java
@@ -94,7 +94,7 @@ public class KNNRegressionExample {
// Save predicted classes to test dataset
for (int i = 0; i < test.rowSize(); i++) {
- double predictedCls = knnMdl.predict(test.getRow(i).features());
+ double predictedCls = knnMdl.apply(test.getRow(i).features());
test.setLabel(i, predictedCls);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/examples/src/main/ml/org/apache/ignite/examples/ml/regression/DistributedRegressionModelExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/regression/DistributedRegressionModelExample.java b/examples/src/main/ml/org/apache/ignite/examples/ml/regression/DistributedRegressionModelExample.java
index ab1b17d..38de97e 100644
--- a/examples/src/main/ml/org/apache/ignite/examples/ml/regression/DistributedRegressionModelExample.java
+++ b/examples/src/main/ml/org/apache/ignite/examples/ml/regression/DistributedRegressionModelExample.java
@@ -123,7 +123,7 @@ public class DistributedRegressionModelExample {
Tracer.showAscii(val);
System.out.println(">>> Trained model prediction results:");
- Tracer.showAscii(mdl.predict(val));
+ Tracer.showAscii(mdl.apply(val));
});
igniteThread.start();
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/Model.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/Model.java b/modules/ml/src/main/java/org/apache/ignite/ml/Model.java
index 05ce774..f0e6cc6 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/Model.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/Model.java
@@ -17,15 +17,11 @@
package org.apache.ignite.ml;
-import java.io.Serializable;
import java.util.function.BiFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
/** Basic interface for all models. */
-@FunctionalInterface
-public interface Model<T, V> extends Serializable {
- /** Predict a result for value. */
- V predict(T val);
-
+public interface Model<T, V> extends IgniteFunction<T, V> {
/**
* Combines this model with other model via specified combiner
*
@@ -34,6 +30,6 @@ public interface Model<T, V> extends Serializable {
* @return Combination of models.
*/
default <X, W> Model<T, X> combine(Model<T, W> other, BiFunction<V, W, X> combiner) {
- return v -> combiner.apply(predict(v), other.predict(v));
+ return v -> combiner.apply(apply(v), other.apply(v));
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansModel.java
index 83fbf1f..70009cb 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansModel.java
@@ -64,7 +64,7 @@ public class FuzzyCMeansModel implements ClusterizationModel<Vector, Integer>, E
* @param val Vector.
* @return Index of the closest center or -1 if it can't be found.
*/
- @Override public Integer predict(Vector val) {
+ @Override public Integer apply(Vector val) {
int idx = -1;
double minDistance = Double.POSITIVE_INFINITY;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java
index 381f976..e1d783f 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java
@@ -65,7 +65,7 @@ public class KMeansModel implements ClusterizationModel<Vector, Integer>, Export
*
* @param vec Vector.
*/
- public Integer predict(Vector vec) {
+ public Integer apply(Vector vec) {
int res = -1;
double minDist = Double.POSITIVE_INFINITY;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/estimators/Estimators.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/estimators/Estimators.java b/modules/ml/src/main/java/org/apache/ignite/ml/estimators/Estimators.java
index 13331d1..b2731ff 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/estimators/Estimators.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/estimators/Estimators.java
@@ -29,7 +29,7 @@ public class Estimators {
/** Simple implementation of mean squared error estimator. */
public static <T, V> IgniteTriFunction<Model<T, V>, Stream<IgniteBiTuple<T, V>>, Function<V, Double>, Double> MSE() {
return (model, stream, f) -> stream.mapToDouble(dp -> {
- double diff = f.apply(dp.get2()) - f.apply(model.predict(dp.get1()));
+ double diff = f.apply(dp.get2()) - f.apply(model.apply(dp.get1()));
return diff * diff;
}).average().orElse(0);
}
@@ -41,7 +41,7 @@ public class Estimators {
long cnt = stream.
peek((ib) -> total.incrementAndGet()).
- filter(dp -> !model.predict(dp.get1()).equals(dp.get2())).
+ filter(dp -> !model.apply(dp.get1()).equals(dp.get2())).
count();
return (double)cnt / total.get();
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModel.java
index 44955c8..d3dff8c 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModel.java
@@ -75,7 +75,7 @@ public class KNNModel implements Model<Vector, Double>, Exportable<KNNModelForma
}
/** {@inheritDoc} */
- @Override public Double predict(Vector v) {
+ @Override public Double apply(Vector v) {
LabeledVector[] neighbors = findKNearestNeighbors(v, true);
return classify(neighbors, v, stgy);
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java
index 90392a3..1796eeb 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java
@@ -43,7 +43,7 @@ public class KNNMultipleLinearRegression extends KNNModel {
}
/** {@inheritDoc} */
- @Override public Double predict(Vector v) {
+ @Override public Double apply(Vector v) {
LabeledVector[] neighbors = findKNearestNeighbors(v, true);
return predictYBasedOn(neighbors, v);
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java
index 6bb0276..7268365 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java
@@ -18,8 +18,10 @@
package org.apache.ignite.ml.math;
import java.util.Map;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
import org.apache.ignite.ml.math.impls.vector.MapWrapperVector;
+import org.apache.ignite.ml.math.impls.vector.SparseDistributedVector;
/**
* Some utils for {@link Vector}.
@@ -39,4 +41,98 @@ public class VectorUtils {
public static Vector fromMap(Map<Integer, Double> val, boolean cp) {
return new MapWrapperVector(val);
}
+
+ /**
+ * Turn number into a local Vector of given size with one-hot encoding.
+ *
+ * @param num Number to turn into vector.
+ * @param vecSize Vector size of output vector.
+ * @return One-hot encoded number.
+ */
+ public static Vector num2Vec(int num, int vecSize) {
+ return num2Vec(num, vecSize, false);
+ }
+
+ /**
+ * Turn number into Vector of given size with one-hot encoding.
+ *
+ * @param num Number to turn into vector.
+ * @param vecSize Vector size of output vector.
+ * @param isDistributed Flag indicating if distributed vector should be created.
+ * @return One-hot encoded number.
+ */
+ public static Vector num2Vec(int num, int vecSize, boolean isDistributed) {
+ Vector res = isDistributed ? new SparseDistributedVector(vecSize) : new DenseLocalOnHeapVector(vecSize);
+ return res.setX(num, 1);
+ }
+
+ /**
+ * Turn Vector into number by looking at index of maximal element in vector.
+ *
+ * @param vec Vector to be turned into number.
+ * @return Number.
+ */
+ public static double vec2Num(Vector vec) {
+ int max = 0;
+ double maxVal = Double.NEGATIVE_INFINITY;
+
+ for (int i = 0; i < vec.size(); i++) {
+ double curVal = vec.getX(i);
+ if (curVal > maxVal) {
+ max = i;
+ maxVal = curVal;
+ }
+ }
+
+ return max;
+ }
+
+ /**
+ * Performs in-place vector multiplication.
+ *
+ * @param vec1 Operand to be changed and first multiplication operand.
+ * @param vec2 Second multiplication operand.
+ * @return Updated first operand.
+ */
+ public static Vector elementWiseTimes(Vector vec1, Vector vec2) {
+ vec1.map(vec2, (a, b) -> a * b);
+
+ return vec1;
+ }
+
+ /**
+ * Performs in-place vector subtraction.
+ *
+ * @param vec1 Operand to be changed and subtracted from.
+ * @param vec2 Operand to subtract.
+ * @return Updated first operand.
+ */
+ public static Vector elementWiseMinus(Vector vec1, Vector vec2) {
+ vec1.map(vec2, (a, b) -> a - b);
+
+ return vec1;
+ }
+
+ /**
+ * Zip two vectors with given binary function
+ * (i.e. apply binary function to both vector elementwise and construct vector from results).
+ *
+ * Example zipWith({0, 2, 4}, {1, 3, 5}, plus) = {0 + 1, 2 + 3, 4 + 5}.
+ * Length of result is length of shortest of vectors.
+ *
+ * @param v1 First vector.
+ * @param v2 Second vector.
+ * @param f Function to zip with.
+ * @return Result of zipping.
+ */
+ public static Vector zipWith(Vector v1, Vector v2, IgniteBiFunction<Double, Double, Double> f) {
+ int size = Math.min(v1.size(), v2.size());
+
+ Vector res = v1.like(size);
+
+ for (int row = 0; row < size; row++)
+ res.setX(row, f.apply(v1.getX(row), v2.getX(row)));
+
+ return res;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableDoubleToDoubleFunction.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableDoubleToDoubleFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableDoubleToDoubleFunction.java
new file mode 100644
index 0000000..e97ee41
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableDoubleToDoubleFunction.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.functions;
+
+/**
+ * Interface for differentiable functions from double to double.
+ */
+public interface IgniteDifferentiableDoubleToDoubleFunction extends IgniteDoubleFunction<Double> {
+ /**
+ * Get function differential at a given point.
+ *
+ * @param pnt Point to calculate differential at.
+ * @return Function differential at a given point.
+ */
+ double differential(double pnt);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableVectorToDoubleFunction.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableVectorToDoubleFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableVectorToDoubleFunction.java
new file mode 100644
index 0000000..3132c63
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableVectorToDoubleFunction.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.functions;
+
+import org.apache.ignite.ml.math.Vector;
+
+/**
+ * Interface for differentiable functions from vector to double.
+ */
+public interface IgniteDifferentiableVectorToDoubleFunction extends IgniteFunction<Vector, Double> {
+ /**
+ * Get function differential at a given point.
+ *
+ * @param pnt Point to calculate differential at.
+ * @return Function differential at a given point.
+ */
+ Vector differential(Vector pnt);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteTriConsumer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteTriConsumer.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteTriConsumer.java
new file mode 100644
index 0000000..af304c9
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteTriConsumer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.functions;
+
+import java.io.Serializable;
+import java.util.function.Consumer;
+
+/**
+ * Serializable tri-consumer.
+ *
+ * @param <A> First parameter type.
+ * @param <B> Second parameter type.
+ * @param <C> Third parameter type.
+ */
+@FunctionalInterface
+public interface IgniteTriConsumer<A, B, C> extends Serializable {
+ /**
+ * Analogous to 'accept' in {@link Consumer} version, but with three parameters.
+ *
+ * @param first First parameter.
+ * @param second Second parameter.
+ * @param third Third parameter.
+ */
+ void accept(A first, B second, C third);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
index ef6bc05..5b6c988 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
@@ -996,4 +996,9 @@ public abstract class AbstractMatrix implements Matrix {
return maxAmountOfCols;
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "Matrix [rows=" + rowSize() + ", cols=" + columnSize() + "]";
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java
index 9b670e5..9f14bc7 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java
@@ -22,6 +22,8 @@ import org.apache.ignite.internal.util.GridArgumentCheck;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.StorageConstants;
import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.functions.IgniteTriFunction;
import org.apache.ignite.ml.math.impls.matrix.CacheMatrix;
import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
import org.apache.ignite.ml.math.impls.matrix.MatrixView;
@@ -209,6 +211,66 @@ public class MatrixUtil {
mtx.setX(i, j, fArr[!isRowMode ? i * colsCnt + j : j * rowsCnt + i]);
}
+ /**
+ * Zip two vectors with given tri-function taking as third argument position in vector
+ * (i.e. apply binary function to both vector elementwise and construct vector from results).
+ * Example zipWith({200, 400, 600}, {100, 300, 500}, plusAndMultiplyByIndex) = {(200 + 100) * 0, (400 + 300) * 1, (600 + 500) * 3}.
+ * Length of result is length of shortest of vectors.
+ *
+ * @param v1 First vector.
+ * @param v2 Second vector.
+ * @param f Function to zip with.
+ * @return Result of zipping.
+ */
+ public static Vector zipWith(Vector v1, Vector v2, IgniteTriFunction<Double, Double, Integer, Double> f) {
+ int size = Math.min(v1.size(), v2.size());
+
+ Vector res = v1.like(size);
+
+ for (int row = 0; row < size; row++)
+ res.setX(row, f.apply(v1.getX(row), v2.getX(row), row));
+
+ return res;
+ }
+
+ /**
+ * Zips two matrices by column-by-column with specified function. Result is matrix same flavour as first matrix.
+ *
+ * @param mtx1 First matrix.
+ * @param mtx2 Second matrix.
+ * @param fun Function to zip with.
+ * @return Vector consisting from values resulted from zipping column-by-column.
+ */
+ public static Vector zipFoldByColumns(Matrix mtx1, Matrix mtx2, IgniteBiFunction<Vector, Vector, Double> fun) {
+ int cols = Math.min(mtx1.columnSize(), mtx2.columnSize());
+
+ Vector vec = mtx1.likeVector(cols);
+
+ for (int i = 0; i < cols; i++)
+ vec.setX(i, fun.apply(mtx1.getCol(i), mtx2.getCol(i)));
+
+ return vec;
+ }
+
+ /**
+ * Zips two matrices by row-by-row with specified function. Result is matrix same flavour as first matrix.
+ *
+ * @param mtx1 First matrix.
+ * @param mtx2 Second matrix.
+ * @param fun Function to zip with.
+ * @return Vector consisting from values resulted from zipping row-by-row.
+ */
+ public static Vector zipFoldByRows(Matrix mtx1, Matrix mtx2, IgniteBiFunction<Vector, Vector, Double> fun) {
+ int rows = Math.min(mtx1.rowSize(), mtx2.rowSize());
+
+ Vector vec = mtx1.likeVector(rows);
+
+ for (int i = 0; i < rows; i++)
+ vec.setX(i, fun.apply(mtx1.viewRow(i), mtx2.viewRow(i)));
+
+ return vec;
+ }
+
/** */
public static double[] flatten(double[][] arr, int stoMode) {
assert arr != null;
@@ -231,4 +293,30 @@ public class MatrixUtil {
return res;
}
+
+ /**
+ * Performs in-place matrix subtraction.
+ *
+ * @param mtx1 Operand to be changed and subtracted from.
+ * @param mtx2 Operand to subtract.
+ * @return Updated first operand.
+ */
+ public static Matrix elementWiseMinus(Matrix mtx1, Matrix mtx2) {
+ mtx1.map(mtx2, (a, b) -> a - b);
+
+ return mtx1;
+ }
+
+ /**
+ * Performs in-place matrix multiplication.
+ *
+ * @param mtx1 Operand to be changed and first multiplication operand.
+ * @param mtx2 Second multiplication operand.
+ * @return Updated first operand.
+ */
+ public static Matrix elementWiseTimes(Matrix mtx1, Matrix mtx2) {
+ mtx1.map(mtx2, (a, b) -> a * b);
+
+ return mtx1;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java
new file mode 100644
index 0000000..f05bde8
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn;
+
+import org.apache.ignite.ml.math.functions.IgniteDifferentiableDoubleToDoubleFunction;
+
+/**
+ * Class containing some common activation functions.
+ */
+public class Activators {
+ /**
+ * Sigmoid activation function.
+ */
+ public static IgniteDifferentiableDoubleToDoubleFunction SIGMOID = new IgniteDifferentiableDoubleToDoubleFunction() {
+ /** {@inheritDoc} */
+ @Override public double differential(double pnt) {
+ double v = apply(pnt);
+ return v * (1 - v);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Double apply(double val) {
+ return 1 / (1 + Math.exp(-val));
+ }
+ };
+
+ /**
+ * Rectified linear unit (ReLU) activation function.
+ */
+ public static IgniteDifferentiableDoubleToDoubleFunction RELU = new IgniteDifferentiableDoubleToDoubleFunction() {
+ /**
+ * Differential of ReLU at pnt. Formally, function is not differentiable at 0, but we let differential at 0 be 0.
+ *
+ * @param pnt Point to differentiate at.
+ * @return Differential at pnt.
+ */
+ @Override public double differential(double pnt) {
+ return pnt > 0 ? 1 : 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Double apply(double val) {
+ return Math.max(val, 0);
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/LocalBatchTrainerInput.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/LocalBatchTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/LocalBatchTrainerInput.java
new file mode 100644
index 0000000..4574841
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/LocalBatchTrainerInput.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn;
+
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.ml.Model;
+import org.apache.ignite.ml.math.Matrix;
+
+/**
+ * Interface for classes containing input parameters for LocalBatchTrainer.
+ */
+public interface LocalBatchTrainerInput<M extends Model<Matrix, Matrix>> {
+ /**
+ * Get next batch in form of matrix of inputs and matrix of outputs.
+ *
+ * @return Next batch.
+ */
+ IgniteBiTuple<Matrix, Matrix> getBatch();
+
+ /**
+ * Model to train.
+ *
+ * @return Model to train.
+ */
+ M mdl();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/LossFunctions.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/LossFunctions.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/LossFunctions.java
new file mode 100644
index 0000000..652072c
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/LossFunctions.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn;
+
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+
+/**
+ * Class containing popular loss functions.
+ */
+public class LossFunctions {
+ /**
+ * Mean squared error loss function.
+ */
+ public static IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> MSE = groundTruth ->
+ new IgniteDifferentiableVectorToDoubleFunction() {
+ /** {@inheritDoc} */
+ @Override public Vector differential(Vector pnt) {
+ double multiplier = 2.0 / pnt.size();
+ return pnt.minus(groundTruth).times(multiplier);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Double apply(Vector vector) {
+ return groundTruth.copy().map(vector, (a, b) -> {
+ double diff = a - b;
+ return diff * diff;
+ }).sum() / (vector.size());
+ }
+ };
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPLayer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPLayer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPLayer.java
new file mode 100644
index 0000000..621dc9f
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPLayer.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn;
+
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.Vector;
+
+/**
+ * Class containing information about layer.
+ */
+public class MLPLayer {
+ /**
+ * Weights matrix.
+ */
+ protected Matrix weights;
+
+ /**
+ * Biases vector.
+ */
+ protected Vector biases;
+
+ /**
+ * Construct MLPLayer from weights and biases.
+ *
+ * @param weights Weights.
+ * @param biases Biases.
+ */
+ public MLPLayer(Matrix weights, Vector biases) {
+ this.weights = weights;
+ this.biases = biases;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPState.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPState.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPState.java
new file mode 100644
index 0000000..5884543
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPState.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.ml.math.Matrix;
+
+/**
+ * State of MLP after computation.
+ */
+public class MLPState {
+ /**
+ * Output of activators.
+ */
+ protected List<Matrix> activatorsOutput;
+
+ /**
+ * Output of linear transformations.
+ */
+ protected List<Matrix> linearOutput;
+
+ /**
+ * Input.
+ */
+ protected Matrix input;
+
+ /**
+ * Construct MLP state.
+ *
+ * @param input Matrix of inputs.
+ */
+ public MLPState(Matrix input) {
+ this.input = input != null ? input.copy() : null;
+ linearOutput = new ArrayList<>();
+ activatorsOutput = new ArrayList<>();
+ }
+
+ /**
+ * Output of activators of given layer. If layer index is 0, inputs are returned.
+ *
+ * @param layer Index of layer to get activators outputs from.
+ * @return Activators output.
+ */
+ public Matrix activatorsOutput(int layer) {
+ return layer > 0 ? activatorsOutput.get(layer - 1) : input;
+ }
+
+ /**
+ * Output of linear transformation of given layer. If layer index is 0, inputs are returned.
+ *
+ * @param layer Index of layer to get linear transformation outputs from.
+ * @return Linear transformation output.
+ */
+ public Matrix linearOutput(int layer) {
+ return layer == 0 ? input : linearOutput.get(layer - 1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java
new file mode 100644
index 0000000..e372d96
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.ml.Model;
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.functions.IgniteDifferentiableDoubleToDoubleFunction;
+import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
+import org.apache.ignite.ml.nn.architecture.TransformationLayerArchitecture;
+import org.apache.ignite.ml.nn.initializers.MLPInitializer;
+import org.apache.ignite.ml.nn.initializers.RandomInitializer;
+import org.apache.ignite.ml.nn.updaters.SmoothParametrized;
+
+import static org.apache.ignite.ml.math.util.MatrixUtil.elementWiseTimes;
+
+/**
+ * Class encapsulating logic of multilayer perceptron.
+ */
+public class MultilayerPerceptron implements Model<Matrix, Matrix>, SmoothParametrized<MultilayerPerceptron> {
+ /**
+ * This MLP architecture.
+ */
+ protected MLPArchitecture architecture;
+
+ /**
+ * List containing layers parameters.
+ */
+ protected List<MLPLayer> layers;
+
+ /**
+ * MLP which is 'below' this MLP (i.e. below output goes to this MLP as input).
+ */
+ protected MultilayerPerceptron below;
+
+ /**
+ * Construct MLP from given architecture and parameters initializer.
+ *
+ * @param arch Architecture.
+ * @param initializer Parameters initializer.
+ */
+ public MultilayerPerceptron(MLPArchitecture arch, MLPInitializer initializer) {
+ layers = new ArrayList<>(arch.layersCount() + 1);
+ architecture = arch;
+ below = null;
+
+ initLayers(initializer);
+ }
+
+ /**
+ * Construct MLP from given architecture.
+ *
+ * @param arch Architecture.
+ */
+ public MultilayerPerceptron(MLPArchitecture arch) {
+ layers = new ArrayList<>(arch.layersCount() + 1);
+ architecture = arch;
+ below = null;
+
+ initLayers(new RandomInitializer(new Random()));
+ }
+
+ /**
+ * Init layers parameters with initializer.
+ *
+ * @param initializer Parameters initializer.
+ */
+ private void initLayers(MLPInitializer initializer) {
+ int prevSize = architecture.inputSize();
+
+ for (int i = 1; i < architecture.layersCount(); i++) {
+ TransformationLayerArchitecture layerCfg = architecture.transformationLayerArchitecture(i);
+ int neuronsCnt = layerCfg.neuronsCount();
+ DenseLocalOnHeapMatrix weights = new DenseLocalOnHeapMatrix(neuronsCnt, prevSize);
+ initializer.initWeights(weights);
+ DenseLocalOnHeapVector biases = null;
+ if (layerCfg.hasBias()) {
+ biases = new DenseLocalOnHeapVector(neuronsCnt);
+ initializer.initBiases(biases);
+ }
+ layers.add(new MLPLayer(weights, biases));
+ prevSize = layerCfg.neuronsCount();
+ }
+ }
+
+ /**
+ * Create MLP from two MLPs: first stacked on second.
+ *
+ * @param above MLP to be above.
+ * @param below MLP to be below.
+ */
+ protected MultilayerPerceptron(MultilayerPerceptron above, MultilayerPerceptron below) {
+ this.layers = above.layers;
+ this.architecture = above.architecture;
+ this.below = below;
+ }
+
+ /**
+ * Perform forward pass and return state of outputs of each layer.
+ *
+ * @param val Value to perform computation on.
+ * @return MLP state after computation.
+ */
+ public MLPState computeState(Matrix val) {
+ MLPState res = new MLPState(val);
+
+ forwardPass(val, res, true);
+
+ return res;
+ }
+
+ /**
+ * Perform forward pass and if needed write state of outputs of each layer.
+ *
+ * @param val Value to perform computation on.
+ * @param state State object to write state into.
+ * @param writeState Flag indicating need to write state.
+ */
+ public Matrix forwardPass(Matrix val, MLPState state, boolean writeState) {
+ Matrix res = val;
+
+ if (below != null)
+ res = below.forwardPass(val, state, writeState);
+
+ for (int i = 1; i < architecture.layersCount(); i++) {
+ MLPLayer curLayer = layers.get(i - 1);
+ res = curLayer.weights.times(res);
+
+ TransformationLayerArchitecture layerCfg = this.architecture.transformationLayerArchitecture(i);
+
+ if (layerCfg.hasBias()) {
+ ReplicatedVectorMatrix biasesMatrix = new ReplicatedVectorMatrix(biases(i), res.columnSize(), true);
+ res = res.plus(biasesMatrix);
+ }
+
+ state.linearOutput.add(res);
+
+ // If we do not write state, we can overwrite result.
+ if (writeState)
+ res = res.copy();
+
+ res = res.map(layerCfg.activationFunction());
+
+ state.activatorsOutput.add(res);
+ }
+
+ return res;
+ }
+
+ /**
+ * Predict values on inputs given as columns in a given matrix.
+ *
+ * @param val Matrix containing inputs as columns.
+ * @return Matrix with predicted vectors stored in columns with column indexes corresponding to column indexes in
+ * the input matrix.
+ */
+ @Override public Matrix apply(Matrix val) {
+ MLPState state = new MLPState(null);
+ forwardPass(val, state, false);
+ return state.activatorsOutput.get(state.activatorsOutput.size() - 1);
+ }
+
+ /**
+ * Create MLP where this MLP output is fed as input to added MLP.
+ *
+ * @param above Added MLP.
+ * @return New MLP where this MLP output is fed as input to added MLP.
+ */
+ public MultilayerPerceptron add(MultilayerPerceptron above) {
+ return new MultilayerPerceptron(above, this);
+ }
+
+ /**
+ * Get weights of layer with given index. Proper indexes are in [1, layersCount).
+ *
+ * @param layerIdx Layer index.
+ * @return Weights of layer with given index.
+ */
+ public Matrix weights(int layerIdx) {
+ assert layerIdx >= 1;
+ assert layerIdx < architecture.layersCount() || below != null;
+
+ if (layerIdx < belowLayersCount())
+ return below.weights(layerIdx - architecture.layersCount());
+ else
+ return layers.get(layerIdx - belowLayersCount() - 1).weights;
+ }
+
+ /**
+ * Get biases of layer with given index. Proper indexes are in [1, layersCount).
+ *
+ * @param layerIdx Layer index.
+ * @return Biases of layer with given index.
+ */
+ public Vector biases(int layerIdx) {
+ assert layerIdx >= 0;
+ assert layerIdx < architecture.layersCount() || below != null;
+
+ if (layerIdx < belowLayersCount())
+ return below.biases(layerIdx - architecture.layersCount());
+ else
+ return layers.get(layerIdx - belowLayersCount() - 1).biases;
+ }
+
+ /**
+ * Checks if layer with given index has biases.
+ *
+ * @param layerIdx Layer index.
+ * @return Value of predicate 'layer with layerIdx has biases'.
+ */
+ public boolean hasBiases(int layerIdx) {
+ return layerIdx != 0 && biases(layerIdx) != null;
+
+ }
+
+ /**
+ * Sets the biases of layer with a given index.
+ *
+ * @param layerIdx Layer index.
+ * @param bias New values for biases.
+ * @return This MLP with updated biases.
+ */
+ public MultilayerPerceptron setBiases(int layerIdx, Vector bias) {
+ biases(layerIdx).assign(bias);
+
+ return this;
+ }
+
+ /**
+ * Set the bias of given neuron in given layer.
+ *
+ * @param layerIdx Layer index.
+ * @param neuronIdx Neuron index.
+ * @param val New value of bias.
+ * @return This MLP with updated biases.
+ */
+ public MultilayerPerceptron setBias(int layerIdx, int neuronIdx, double val) {
+ // Should be transformation layer.
+ assert layerIdx > 0;
+ assert architecture.transformationLayerArchitecture(layerIdx).hasBias();
+
+ biases(layerIdx).setX(neuronIdx, val);
+
+ return this;
+ }
+
+ /**
+ * Get the bias of given neuron in given layer.
+ *
+ * @param layerIdx Layer index.
+ * @param neuronIdx Neuron index.
+ * @return Bias with specified coordinates.
+ */
+ public double bias(int layerIdx, int neuronIdx) {
+ // Should be transformation layer.
+ assert layerIdx > 0;
+ assert architecture.transformationLayerArchitecture(layerIdx).hasBias();
+
+ return biases(layerIdx).getX(neuronIdx);
+ }
+
+ /**
+ * Sets the weighs of layer with a given index.
+ *
+ * @param layerIdx Layer index.
+ * @param weights New values for weights.
+ * @return This MLP with updated weights.
+ */
+ public MultilayerPerceptron setWeights(int layerIdx, Matrix weights) {
+ weights(layerIdx).assign(weights);
+
+ return this;
+ }
+
+ /**
+ * Set the weight of neuron with given index in previous layer to neuron with given index in given layer.
+ *
+ * @param layerIdx Layer index.
+ * @param fromNeuron Neuron index in previous layer.
+ * @param toNeuron Neuron index in current layer.
+ * @param val New value of weight.
+ * @return This MLP with updated weights.
+ */
+ public MultilayerPerceptron setWeight(int layerIdx, int fromNeuron, int toNeuron, double val) {
+ // Should be transformation layer.
+ assert layerIdx > 0;
+
+ weights(layerIdx).setX(toNeuron, fromNeuron, val);
+
+ return this;
+ }
+
+ /**
+ * Get the weight of neuron with given index in previous layer to neuron with given index in given layer.
+ *
+ * @param layerIdx Layer index.
+ * @param fromNeuron Neuron index in previous layer.
+ * @param toNeuron Neuron index in current layer.
+ * @return Weight with specified coordinates.
+ */
+ public double weight(int layerIdx, int fromNeuron, int toNeuron) {
+ // Should be transformation layer.
+ assert layerIdx > 0;
+ assert architecture.transformationLayerArchitecture(layerIdx).hasBias();
+
+ return weights(layerIdx).getX(fromNeuron, toNeuron);
+ }
+
+ /**
+ * Get count of layers in this MLP.
+ *
+ * @return Count of layers in this MLP.
+ */
+ public int layersCount() {
+ return architecture.layersCount() + (below != null ? below.layersCount() : 0);
+ }
+
+ /** Count of layers in below MLP. */
+ protected int belowLayersCount() {
+ return below != null ? below.layersCount() : 0;
+ }
+
+ /**
+ * Get architecture of this MLP.
+ *
+ * @return Architecture of this MLP.
+ */
+ public MLPArchitecture architecture() {
+ if (below != null)
+ return below.architecture().add(architecture());
+ return architecture;
+ }
+
+ /** {@inheritDoc} */
+ public Vector differentiateByParameters(IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss,
+ Matrix inputsBatch, Matrix truthBatch) {
+ // Backpropagation algorithm is used here.
+ int batchSize = inputsBatch.columnSize();
+ double invBatchSize = 1 / (double)batchSize;
+ int lastLayer = layersCount() - 1;
+ MLPState mlpState = computeState(inputsBatch);
+ Matrix dz = null;
+
+ List<MLPLayer> layersParameters = new LinkedList<>();
+
+ for (int layer = lastLayer; layer > 0; layer--) {
+ Matrix z = mlpState.linearOutput(layer).copy();
+ Matrix dSigmaDz = differentiateNonlinearity(z,
+ architecture().transformationLayerArchitecture(layer).activationFunction());
+
+ if (layer == lastLayer) {
+ Matrix sigma = mlpState.activatorsOutput(lastLayer).copy();
+ Matrix dLossDSigma = differentiateLoss(truthBatch, sigma, loss);
+ dz = elementWiseTimes(dLossDSigma, dSigmaDz);
+ }
+ else {
+ dz = weights(layer + 1).transpose().times(dz);
+ dz = elementWiseTimes(dz, dSigmaDz);
+ }
+
+ Matrix a = mlpState.activatorsOutput(layer - 1);
+ Matrix dw = dz.times(a.transpose()).times(invBatchSize);
+
+ Vector db = null;
+ if (hasBiases(layer))
+ db = dz.foldRows(Vector::sum).times(invBatchSize);
+
+ // Because we go from last layer, add each layer to the begining.
+ layersParameters.add(0, new MLPLayer(dw, db));
+ }
+
+ return paramsAsVector(layersParameters);
+ }
+
+ /** {@inheritDoc} */
+ public Vector parameters() {
+ return paramsAsVector(layers);
+ }
+
+ /**
+ * Flatten this MLP parameters as vector.
+ *
+ * @param layersParams List of layers parameters.
+ * @return This MLP parameters as vector.
+ */
+ protected Vector paramsAsVector(List<MLPLayer> layersParams) {
+ int off = 0;
+ Vector res = new DenseLocalOnHeapVector(architecture().parametersCount());
+
+ for (MLPLayer layerParams : layersParams) {
+ off = writeToVector(res, layerParams.weights, off);
+
+ if (layerParams.biases != null)
+ off = writeToVector(res, layerParams.biases, off);
+ }
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ public MultilayerPerceptron setParameters(Vector vector) {
+ int off = 0;
+
+ for (int l = 1; l < layersCount(); l++) {
+ MLPLayer layer = layers.get(l - 1);
+
+ IgniteBiTuple<Integer, Matrix> readRes = readFromVector(vector, layer.weights.rowSize(),
+ layer.weights.columnSize(), off);
+
+ off = readRes.get1();
+ layer.weights = readRes.get2();
+
+ if (hasBiases(l)) {
+ IgniteBiTuple<Integer, Vector> readRes1 = readFromVector(vector, layer.biases.size(), off);
+ off = readRes1.get1();
+
+ layer.biases = readRes1.get2();
+ }
+ }
+
+ return this;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int parametersCount() {
+ return architecture().parametersCount();
+ }
+
+ /**
+ * Read matrix with given dimensions from vector starting with offset and return new offset position
+ * which is last matrix entry position + 1.
+ *
+ * @param v Vector to read from.
+ * @param rows Count of rows of matrix to read.
+ * @param cols Count of columns of matrix to read.
+ * @param off Start read position.
+ * @return New offset position which is last matrix entry position + 1.
+ */
+ private IgniteBiTuple<Integer, Matrix> readFromVector(Vector v, int rows, int cols, int off) {
+ Matrix mtx = new DenseLocalOnHeapMatrix(rows, cols);
+
+ int size = rows * cols;
+ for (int i = 0; i < size; i++)
+ mtx.setX(i / cols, i % cols, v.getX(off + i));
+
+ return new IgniteBiTuple<>(off + size, mtx);
+ }
+
+ /**
+ * Read vector of given size from vector and return new offset position which is last read vector entry position + 1.
+ *
+ * @param v Vector to read from.
+ * @param size Size of vector to read.
+ * @param off Start read position.
+ * @return New offset position which is last read vector entry position + 1.
+ */
+ private IgniteBiTuple<Integer, Vector> readFromVector(Vector v, int size, int off) {
+ Vector vec = new DenseLocalOnHeapVector(size);
+
+ for (int i = 0; i < size; i++)
+ vec.setX(i, v.getX(off + i));
+
+ return new IgniteBiTuple<>(off + size, vec);
+ }
+
+ /**
+ * Write matrix into vector starting from offset and return new offset position which is last written entry position + 1.
+ *
+ * @param vec Vector to write into.
+ * @param mtx Matrix to write.
+ * @param off Start write position.
+ * @return New offset position which is last written entry position + 1.
+ */
+ private int writeToVector(Vector vec, Matrix mtx, int off) {
+ int rows = mtx.rowSize();
+ int cols = mtx.columnSize();
+
+ for (int r = 0; r < rows; r++) {
+ for (int c = 0; c < cols; c++) {
+ vec.setX(off, mtx.getX(r, c));
+ off++;
+ }
+ }
+
+ return off;
+ }
+
+ /**
+ * Write vector into vector starting from offset and return new offset position which is last written entry position + 1.
+ *
+ * @param vec Vector to write into.
+ * @param v Vector to write.
+ * @param off Start write position.
+ * @return New offset position which is last written entry position + 1.
+ */
+ private int writeToVector(Vector vec, Vector v, int off) {
+ for (int i = 0; i < v.size(); i++) {
+ vec.setX(off, v.getX(i));
+ off++;
+ }
+
+ return off;
+ }
+
+ /**
+ * Differentiate loss.
+ *
+ * @param groundTruth Ground truth values.
+ * @param lastLayerOutput Last layer output.
+ * @param loss Loss function.
+ * @return Gradients matrix.
+ */
+ private Matrix differentiateLoss(Matrix groundTruth, Matrix lastLayerOutput,
+ IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss) {
+ Matrix diff = groundTruth.like(groundTruth.rowSize(), groundTruth.columnSize());
+
+ for (int col = 0; col < groundTruth.columnSize(); col++) {
+ // TODO: IGNITE-7155 Couldn't use views here because copy on views doesn't do actual copy and all changes are propagated to original.
+ Vector gtCol = groundTruth.getCol(col);
+ Vector predCol = lastLayerOutput.getCol(col);
+ diff.assignColumn(col, loss.apply(gtCol).differential(predCol));
+ }
+
+ return diff;
+ }
+
+ /**
+ * Differentiate nonlinearity.
+ *
+ * @param linearOut Linear output of current layer.
+ * @param nonlinearity Nonlinearity of current layer.
+ * @return Gradients matrix.
+ */
+ private Matrix differentiateNonlinearity(Matrix linearOut, IgniteDifferentiableDoubleToDoubleFunction nonlinearity) {
+ Matrix diff = linearOut.copy();
+
+ diff.map(nonlinearity::differential);
+
+ return diff;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/ReplicatedVectorMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/ReplicatedVectorMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/ReplicatedVectorMatrix.java
new file mode 100644
index 0000000..559206d
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/ReplicatedVectorMatrix.java
@@ -0,0 +1,583 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Map;
+import java.util.Spliterator;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.MatrixStorage;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.exceptions.CardinalityException;
+import org.apache.ignite.ml.math.functions.IgniteBiConsumer;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.functions.IgniteDoubleFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.functions.IgniteTriFunction;
+import org.apache.ignite.ml.math.functions.IntIntToDoubleFunction;
+import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
+
+/**
+ * Convenient way to create matrix of replicated columns or rows from vector.
+ * This class should be considered as utility class: not all matrix methods are implemented here, only those which
+ * were necessary for MLPs.
+ */
+class ReplicatedVectorMatrix implements Matrix {
+ /**
+ * Vector to replicate.
+ */
+ private Vector vector;
+
+ /**
+ * Flag determining is vector replicated as column or row.
+ */
+ private boolean asCol;
+
+ /**
+ * Count of vector replications.
+ */
+ private int replicationCnt;
+
+ /**
+ * Construct ReplicatedVectorMatrix.
+ *
+ * @param vector Vector to replicate.
+ * @param replicationCnt Count of replications.
+ * @param asCol Should vector be replicated as a column or as a row.
+ */
+ ReplicatedVectorMatrix(Vector vector, int replicationCnt, boolean asCol) {
+ this.vector = vector;
+ this.asCol = asCol;
+ this.replicationCnt = replicationCnt;
+ }
+
+ /**
+ * Constructor for externalization.
+ */
+ public ReplicatedVectorMatrix() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isSequentialAccess() {
+ return vector.isSequentialAccess();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isRandomAccess() {
+ return vector.isRandomAccess();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isDense() {
+ return vector.isDense();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isArrayBased() {
+ return vector.isArrayBased();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isDistributed() {
+ return vector.isDistributed();
+ }
+
+ /** {@inheritDoc} */
+ @Override public double maxValue() {
+ return vector.maxValue();
+ }
+
+ /** {@inheritDoc} */
+ @Override public double minValue() {
+ return vector.minValue();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Element maxElement() {
+ return new Element() {
+ @Override public double get() {
+ return vector.maxElement().get();
+ }
+
+ @Override public int row() {
+ return asCol ? vector.maxElement().index() : 0;
+ }
+
+ @Override public int column() {
+ return asCol ? 0 : vector.maxElement().index();
+ }
+
+ @Override public void set(double val) {
+
+ }
+ };
+ }
+
+ /** {@inheritDoc} */
+ @Override public Element minElement() {
+ return new Element() {
+ @Override public double get() {
+ return vector.minElement().get();
+ }
+
+ @Override public int row() {
+ return asCol ? vector.minElement().index() : 0;
+ }
+
+ @Override public int column() {
+ return asCol ? 0 : vector.minElement().index();
+ }
+
+ @Override public void set(double val) {
+
+ }
+ };
+ }
+
+ /** {@inheritDoc} */
+ @Override public Element getElement(int row, int col) {
+ Vector.Element el = asCol ? vector.getElement(row) : vector.getElement(col);
+ int r = asCol ? el.index() : 0;
+ int c = asCol ? 0 : el.index();
+
+ return new Element() {
+ @Override public double get() {
+ return el.get();
+ }
+
+ @Override public int row() {
+ return r;
+ }
+
+ @Override public int column() {
+ return c;
+ }
+
+ @Override public void set(double val) {
+
+ }
+ };
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix swapRows(int row1, int row2) {
+ return asCol ? new ReplicatedVectorMatrix(swap(row1, row2), replicationCnt, asCol) : this;
+ }
+
+ /** {@inheritDoc} */
+ private Vector swap(int idx1, int idx2) {
+ double val = vector.getX(idx1);
+
+ vector.setX(idx1, vector.getX(idx2));
+ vector.setX(idx2, val);
+
+ return vector;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix swapColumns(int col1, int col2) {
+ return asCol ? this : new ReplicatedVectorMatrix(swap(col1, col2), replicationCnt, asCol);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix assign(double val) {
+ return new ReplicatedVectorMatrix(vector.assign(val), replicationCnt, asCol);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix assign(double[][] vals) {
+ return new DenseLocalOnHeapMatrix(vals);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix assign(Matrix mtx) {
+ return mtx.copy();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix assign(IntIntToDoubleFunction fun) {
+ Vector vec = asCol ? this.vector.assign(idx -> fun.apply(idx, 0)) : this.vector.assign(idx -> fun.apply(0, idx));
+ return new ReplicatedVectorMatrix(vec, replicationCnt, asCol);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix map(IgniteDoubleFunction<Double> fun) {
+ Vector vec = vector.map(fun);
+ return new ReplicatedVectorMatrix(vec, replicationCnt, asCol);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix map(Matrix mtx, IgniteBiFunction<Double, Double, Double> fun) {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int nonZeroElements() {
+ return vector.nonZeroElements() * (asCol ? columnSize() : rowSize());
+ }
+
+ /** {@inheritDoc} */
+ @Override public Spliterator<Double> allSpliterator() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Spliterator<Double> nonZeroSpliterator() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix assignColumn(int col, Vector vec) {
+ int rows = asCol ? vector.size() : replicationCnt;
+ int cols = asCol ? replicationCnt : vector.size();
+ int times = asCol ? cols : rows;
+
+ Matrix res = new DenseLocalOnHeapMatrix(rows, cols);
+
+ IgniteBiConsumer<Integer, Vector> replicantAssigner = asCol ? res::assignColumn : res::assignRow;
+ IgniteBiConsumer<Integer, Vector> assigner = res::assignColumn;
+
+ assign(replicantAssigner, assigner, vector, vec, times, col);
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix assignRow(int row, Vector vec) {
+ int rows = asCol ? vector.size() : replicationCnt;
+ int cols = asCol ? replicationCnt : vector.size();
+ int times = asCol ? cols : rows;
+
+ Matrix res = new DenseLocalOnHeapMatrix(rows, cols);
+
+ IgniteBiConsumer<Integer, Vector> replicantAssigner = asCol ? res::assignColumn : res::assignRow;
+ IgniteBiConsumer<Integer, Vector> assigner = res::assignRow;
+
+ assign(replicantAssigner, assigner, vector, vec, times, row);
+
+ return res;
+ }
+
+ /** */
+ private void assign(IgniteBiConsumer<Integer, Vector> replicantAssigner,
+ IgniteBiConsumer<Integer, Vector> assigner, Vector replicant, Vector vector, int times, int idx) {
+ for (int i = 0; i < times; i++)
+ replicantAssigner.accept(i, replicant);
+ assigner.accept(idx, vector);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Vector foldRows(IgniteFunction<Vector, Double> fun) {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Vector foldColumns(IgniteFunction<Vector, Double> fun) {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T> T foldMap(IgniteBiFunction<T, Double, T> foldFun, IgniteDoubleFunction<Double> mapFun,
+ T zeroVal) {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean density(double threshold) {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int columnSize() {
+ return asCol ? replicationCnt : vector.size();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int rowSize() {
+ return asCol ? vector.size() : replicationCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public double determinant() {
+ // If matrix is not square throw exception.
+ checkCardinality(vector.size(), replicationCnt);
+
+ // If matrix is 1x1 then determinant is its single element otherwise there are linear dependence and determinant is 0.
+ return vector.size() > 0 ? 0 : vector.get(1);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix inverse() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix divide(double x) {
+ return new ReplicatedVectorMatrix(vector.divide(x), replicationCnt, asCol);
+ }
+
+ /** {@inheritDoc} */
+ @Override public double get(int row, int col) {
+ return asCol ? vector.get(row) : vector.get(col);
+ }
+
+ /** {@inheritDoc} */
+ @Override public double getX(int row, int col) {
+ return asCol ? vector.getX(row) : vector.getX(col);
+ }
+
+ /** {@inheritDoc} */
+ @Override public MatrixStorage getStorage() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix copy() {
+ Vector cp = vector.copy();
+ return new ReplicatedVectorMatrix(cp, replicationCnt, asCol);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix like(int rows, int cols) {
+ Vector lk = vector.like(vector.size());
+ return new ReplicatedVectorMatrix(lk, replicationCnt, asCol);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Vector likeVector(int crd) {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix minus(Matrix mtx) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Specialized optimized version of minus for ReplicatedVectorMatrix.
+ *
+ * @param mtx Matrix to be subtracted.
+ * @return new ReplicatedVectorMatrix resulting from subtraction.
+ */
+ public Matrix minus(ReplicatedVectorMatrix mtx) {
+ if (isColumnReplicated() == mtx.isColumnReplicated()) {
+ checkCardinality(mtx.rowSize(), mtx.columnSize());
+
+ Vector minus = vector.minus(mtx.replicant());
+
+ return new ReplicatedVectorMatrix(minus, replicationCnt, asCol);
+ }
+
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix plus(double x) {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix plus(Matrix mtx) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Specialized optimized version of plus for ReplicatedVectorMatrix.
+ *
+ * @param mtx Matrix to be added.
+ * @return new ReplicatedVectorMatrix resulting from addition.
+ */
+ public Matrix plus(ReplicatedVectorMatrix mtx) {
+ if (isColumnReplicated() == mtx.isColumnReplicated()) {
+ checkCardinality(mtx.rowSize(), mtx.columnSize());
+
+ Vector plus = vector.plus(mtx.replicant());
+
+ return new ReplicatedVectorMatrix(plus, replicationCnt, asCol);
+ }
+
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Checks that dimensions of this matrix are equal to given dimensions.
+ *
+ * @param rows Rows.
+ * @param cols Columns.
+ */
+ private void checkCardinality(int rows, int cols) {
+ if (rows != rowSize())
+ throw new CardinalityException(rowSize(), rows);
+
+ if (cols != columnSize())
+ throw new CardinalityException(columnSize(), cols);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid guid() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix set(int row, int col, double val) {
+ vector.set(asCol ? row : col, val);
+
+ return this;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix setRow(int row, double[] data) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Vector getRow(int row) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix setColumn(int col, double[] data) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Vector getCol(int col) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix setX(int row, int col, double val) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix times(double x) {
+ return new ReplicatedVectorMatrix(vector.times(x), replicationCnt, asCol);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix times(Matrix mtx) {
+ if (!asCol) {
+ Vector row = vector.like(mtx.columnSize());
+
+ for (int i = 0; i < mtx.columnSize(); i++)
+ row.setX(i, vector.dot(mtx.getCol(i)));
+
+ return new ReplicatedVectorMatrix(row, replicationCnt, false);
+
+ }
+ else
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Vector times(Vector vec) {
+ Vector res = vec.like(vec.size());
+ if (asCol) {
+ for (int i = 0; i < rowSize(); i++)
+ res.setX(i, vec.sum() * vector.getX(i));
+ }
+ else {
+ double val = vector.dot(vec);
+
+ for (int i = 0; i < rowSize(); i++)
+ res.setX(i, val);
+ }
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public double maxAbsRowSumNorm() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public double sum() {
+ return vector.sum() * replicationCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix transpose() {
+ return new ReplicatedVectorMatrix(vector, replicationCnt, !asCol);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix viewPart(int[] off, int[] size) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix viewPart(int rowOff, int rows, int colOff, int cols) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Vector viewRow(int row) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Vector viewColumn(int col) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Vector viewDiagonal() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void compute(int row, int col, IgniteTriFunction<Integer, Integer, Double, Double> f) {
+ // This operation cannot be performed because computing function depends on both indexes and therefore
+ // result of compute will be in general case not ReplicatedVectorMatrix.
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<String, Object> getMetaStorage() {
+ return null;
+ }
+
+ /**
+ * Returns true if matrix constructed by replicating vector as column and false otherwise.
+ */
+ public boolean isColumnReplicated() {
+ return asCol;
+ }
+
+ /**
+ * Returns replicated vector.
+ */
+ public Vector replicant() {
+ return vector;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/LayerArchitecture.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/LayerArchitecture.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/LayerArchitecture.java
new file mode 100644
index 0000000..31a3e9a
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/LayerArchitecture.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn.architecture;
+
+/**
+ * Layer architecture.
+ */
+public class LayerArchitecture {
+ /**
+ * Count of neurons on layer.
+ */
+ private final int neuronsCnt;
+
+ /**
+ * Construct LayerArchitecture.
+ *
+ * @param neuronsCnt Count of neurons in layer.
+ */
+ public LayerArchitecture(int neuronsCnt) {
+ this.neuronsCnt = neuronsCnt;
+ }
+
+ /**
+ * Get count of neurons in layer.
+ * @return Count of neurons in layer.
+ */
+ public int neuronsCount() {
+ return neuronsCnt;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/MLPArchitecture.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/MLPArchitecture.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/MLPArchitecture.java
new file mode 100644
index 0000000..3ef7b61
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/MLPArchitecture.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.nn.architecture;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.ml.math.functions.IgniteDifferentiableDoubleToDoubleFunction;
+
+/**
+ * Class containing information about architecture of MLP.
+ */
+public class MLPArchitecture {
+ /**
+ * List of layers architectures.
+ */
+ private final List<LayerArchitecture> layers;
+
+ /**
+ * Construct an MLP architecture.
+ *
+ * @param inputSize Size of input to MLP.
+ */
+ public MLPArchitecture(int inputSize) {
+ layers = new ArrayList<>();
+ layers.add(new LayerArchitecture(inputSize));
+ }
+
+ /**
+ * Construct an MLP architecture.
+ *
+ * @param layers List of layers architectures.
+ */
+ private MLPArchitecture(List<LayerArchitecture> layers) {
+ this.layers = layers;
+ }
+
+ /**
+ * Count of layers in MLP.
+ * @return Layers count.
+ */
+ public int layersCount() {
+ return layers.size();
+ }
+
+ /**
+ * Size of input of MLP.
+ * @return Size of input.
+ */
+ public int inputSize() {
+ return layers.get(0).neuronsCount();
+ }
+
+ /**
+ * Size of output of MLP.
+ * @return Size of output.
+ */
+ public int outputSize() {
+ return layers.get(layersCount() - 1).neuronsCount();
+ }
+
+ /**
+ * Constructs new MLP architecture with new layer added on top of all this architecture layers.
+ *
+ * @param neuronsCnt Count of neurons in new layer.
+ * @param hasBias Flag indicating presence of bias in added layer.
+ * @param f Activation function of a new layer.
+ * @return New MLP architecture with new layer added on top of all this architecture layers.
+ */
+ public MLPArchitecture withAddedLayer(int neuronsCnt, boolean hasBias, IgniteDifferentiableDoubleToDoubleFunction f) {
+ ArrayList<LayerArchitecture> newLayers = new ArrayList<>(layers);
+
+ newLayers.add(new TransformationLayerArchitecture(neuronsCnt, hasBias, f));
+
+ return new MLPArchitecture(newLayers);
+ }
+
+ /**
+ * Get architecture of layer with given index.
+ *
+ * @param layer Index of layer to get architecture from.
+ * @return Architecture of layer with given index.
+ */
+ public LayerArchitecture layerArchitecture(int layer) {
+ return layers.get(layer);
+ }
+
+ /**
+ * Get architecture of transformation layer (i.e. non-input layer) with given index.
+ *
+ * @param layer Index of layer to get architecture from.
+ * @return Architecture of transformation layer with given index.
+ */
+ public TransformationLayerArchitecture transformationLayerArchitecture(int layer) {
+ return (TransformationLayerArchitecture)layers.get(layer);
+ }
+
+ /**
+ * Creates config describing network where first goes this config and after goes this method's argument.
+ *
+ * @param second Config to add after this config.
+ * @return New combined configuration.
+ */
+ public MLPArchitecture add(MLPArchitecture second) {
+ assert second.inputSize() == outputSize();
+
+ MLPArchitecture res = new MLPArchitecture(inputSize());
+ res.layers.addAll(layers);
+ res.layers.addAll(second.layers);
+
+ return res;
+ }
+
+ /**
+ * Count of parameters in this MLP architecture.
+ *
+ * @return Parameters in this MLP architecture.
+ */
+ public int parametersCount() {
+ int res = 0;
+
+ for (int i = 1; i < layersCount(); i++) {
+ TransformationLayerArchitecture la = transformationLayerArchitecture(i);
+ res += layerArchitecture(i - 1).neuronsCount() * la.neuronsCount();
+
+ if (la.hasBias())
+ res += la.neuronsCount();
+
+ }
+
+ return res;
+ }
+}