You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2017/08/01 09:26:56 UTC
[11/50] [abbrv] ignite git commit: IGNITE-5791 Block matrix
introduction
IGNITE-5791 Block matrix introduction
This closes #2326
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0d2b989d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0d2b989d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0d2b989d
Branch: refs/heads/ignite-5757
Commit: 0d2b989d2be62533a36061940497a734463b5f10
Parents: db43b0c
Author: Yury Babak <yb...@gridgain.com>
Authored: Fri Jul 21 15:28:21 2017 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Fri Jul 21 15:28:21 2017 +0300
----------------------------------------------------------------------
.../apache/ignite/ml/math/DistanceMeasure.java | 2 +-
.../ignite/ml/math/EuclideanDistance.java | 3 +-
.../math/decompositions/EigenDecomposition.java | 2 +-
.../apache/ignite/ml/math/impls/CacheUtils.java | 198 +++++++--
.../ml/math/impls/matrix/AbstractMatrix.java | 4 +-
.../ignite/ml/math/impls/matrix/BlockEntry.java | 50 +++
.../ml/math/impls/matrix/CacheMatrix.java | 9 +-
.../matrix/SparseBlockDistributedMatrix.java | 208 +++++++++
.../impls/matrix/SparseDistributedMatrix.java | 26 +-
.../storage/matrix/BaseBlockMatrixKey.java | 41 ++
.../impls/storage/matrix/BlockMatrixKey.java | 144 ++++++
.../storage/matrix/BlockMatrixStorage.java | 435 +++++++++++++++++++
.../vector/SparseLocalOnHeapVectorStorage.java | 4 +-
.../ignite/ml/math/statistics/Variance.java | 1 +
.../ignite/ml/math/statistics/package-info.java | 22 +
.../org/apache/ignite/ml/math/util/MapUtil.java | 2 +-
.../ignite/ml/math/util/package-info.java | 22 +
.../java/org/apache/ignite/ml/package-info.java | 22 +
.../ml/math/MathImplDistributedTestSuite.java | 2 +
.../SparseDistributedBlockMatrixTest.java | 379 ++++++++++++++++
.../matrix/SparseDistributedMatrixTest.java | 32 +-
21 files changed, 1528 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java
index 09be0c3..df235a7 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java
@@ -34,5 +34,5 @@ public interface DistanceMeasure extends Externalizable {
* @return the distance between the two vectors
* @throws CardinalityException if the array lengths differ.
*/
- double compute(Vector a, Vector b) throws CardinalityException;
+ public double compute(Vector a, Vector b) throws CardinalityException;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java
index 5f962ce..edc11dc 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java
@@ -30,8 +30,7 @@ public class EuclideanDistance implements DistanceMeasure {
private static final long serialVersionUID = 1717556319784040040L;
/** {@inheritDoc} */
- @Override
- public double compute(Vector a, Vector b)
+ @Override public double compute(Vector a, Vector b)
throws CardinalityException {
return MatrixUtil.localCopyOf(a).minus(b).kNorm(2.0);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/EigenDecomposition.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/EigenDecomposition.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/EigenDecomposition.java
index d0e91a5..a5c92e6 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/EigenDecomposition.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/EigenDecomposition.java
@@ -446,7 +446,7 @@ public class EigenDecomposition implements Destroyable {
// Store roots isolated by balanc and compute matrix norm
- double norm = h.foldMap(Functions.PLUS, Functions.ABS, 0.0);
+ double norm = h.foldMap(Functions.PLUS, Functions.ABS, 0.0d);
// Outer loop over eigenvalue index
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
index 1bda5e6..369840b 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
@@ -39,11 +39,16 @@ import org.apache.ignite.ml.math.KeyMapper;
import org.apache.ignite.ml.math.ValueMapper;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.functions.IgniteConsumer;
+import org.apache.ignite.ml.math.functions.IgniteDoubleFunction;
import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
+import org.apache.ignite.ml.math.impls.matrix.BlockEntry;
+import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixKey;
+import org.apache.ignite.internal.util.typedef.internal.A;
/**
* Distribution-related misc. support.
+ *
+ * TODO: IGNITE-5102, fix sparse key filters
*/
public class CacheUtils {
/**
@@ -127,19 +132,38 @@ public class CacheUtils {
* @param matrixUuid Matrix UUID.
* @return Sum obtained using sparse logic.
*/
- public static <K, V> double sparseSum(IgniteUuid matrixUuid) {
- Collection<Double> subSums = fold(SparseDistributedMatrixStorage.ML_CACHE_NAME, (CacheEntry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> ce, Double acc) -> {
- Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> entry = ce.entry();
- if (entry.getKey().get2().equals(matrixUuid)) {
- Map<Integer, Double> map = entry.getValue();
+ @SuppressWarnings("unchecked")
+ public static <K, V> double sparseSum(IgniteUuid matrixUuid, String cacheName) {
+ A.notNull(matrixUuid, "matrixUuid");
+ A.notNull(cacheName, "cacheName");
+
+ Collection<Double> subSums = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
+ V v = ce.entry().getValue();
+
+ double sum = 0.0;
- double sum = sum(map.values());
+ if (v instanceof Map) {
+ Map<Integer, Double> map = (Map<Integer, Double>)v;
- return acc == null ? sum : acc + sum;
+ sum = sum(map.values());
+ }
+ else if (v instanceof BlockEntry) {
+ BlockEntry be = (BlockEntry)v;
+
+ sum = be.sum();
}
else
- return acc;
- }, key -> key.get2().equals(matrixUuid));
+ throw new UnsupportedOperationException();
+
+ return acc == null ? sum : acc + sum;
+ }, key -> {
+ if (key instanceof BlockMatrixKey)
+ return ((BlockMatrixKey)key).matrixId().equals(matrixUuid);
+ else if (key instanceof IgniteBiTuple)
+ return ((IgniteBiTuple<Integer, IgniteUuid>)key).get2().equals(matrixUuid);
+ else
+ throw new UnsupportedOperationException();
+ });
return sum(subSums);
}
@@ -186,23 +210,42 @@ public class CacheUtils {
* @param matrixUuid Matrix UUID.
* @return Minimum value obtained using sparse logic.
*/
- public static <K, V> double sparseMin(IgniteUuid matrixUuid) {
- Collection<Double> mins = fold(SparseDistributedMatrixStorage.ML_CACHE_NAME, (CacheEntry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> ce, Double acc) -> {
- Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> entry = ce.entry();
+ @SuppressWarnings("unchecked")
+ public static <K, V> double sparseMin(IgniteUuid matrixUuid, String cacheName) {
+ A.notNull(matrixUuid, "matrixUuid");
+ A.notNull(cacheName, "cacheName");
- if (entry.getKey().get2().equals(matrixUuid)) {
- Map<Integer, Double> map = entry.getValue();
+ Collection<Double> mins = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
+ V v = ce.entry().getValue();
- double min = Collections.min(map.values());
+ double min;
- if (acc == null)
- return min;
- else
- return Math.min(acc, min);
+ if (v instanceof Map) {
+ Map<Integer, Double> map = (Map<Integer, Double>)v;
+
+ min = Collections.min(map.values());
+ }
+ else if (v instanceof BlockEntry) {
+ BlockEntry be = (BlockEntry)v;
+
+ min = be.minValue();
}
else
- return acc;
- }, key -> key.get2().equals(matrixUuid));
+ throw new UnsupportedOperationException();
+
+ if (acc == null)
+ return min;
+ else
+ return Math.min(acc, min);
+
+ }, key -> {
+ if (key instanceof BlockMatrixKey)
+ return ((BlockMatrixKey)key).matrixId().equals(matrixUuid);
+ else if (key instanceof IgniteBiTuple)
+ return ((IgniteBiTuple<Integer, IgniteUuid>)key).get2().equals(matrixUuid);
+ else
+ throw new UnsupportedOperationException();
+ });
return Collections.min(mins);
}
@@ -211,22 +254,42 @@ public class CacheUtils {
* @param matrixUuid Matrix UUID.
* @return Maximum value obtained using sparse logic.
*/
- public static <K, V> double sparseMax(IgniteUuid matrixUuid) {
- Collection<Double> maxes = fold(SparseDistributedMatrixStorage.ML_CACHE_NAME, (CacheEntry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> ce, Double acc) -> {
- Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> entry = ce.entry();
- if (entry.getKey().get2().equals(matrixUuid)) {
- Map<Integer, Double> map = entry.getValue();
+ @SuppressWarnings("unchecked")
+ public static <K, V> double sparseMax(IgniteUuid matrixUuid, String cacheName) {
+ A.notNull(matrixUuid, "matrixUuid");
+ A.notNull(cacheName, "cacheName");
+
+ Collection<Double> maxes = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
+ V v = ce.entry().getValue();
- double max = Collections.max(map.values());
+ double max;
- if (acc == null)
- return max;
- else
- return Math.max(acc, max);
+ if (v instanceof Map) {
+ Map<Integer, Double> map = (Map<Integer, Double>)v;
+
+ max = Collections.max(map.values());
+ }
+ else if (v instanceof BlockEntry) {
+ BlockEntry be = (BlockEntry)v;
+
+ max = be.maxValue();
}
else
- return acc;
- }, key -> key.get2().equals(matrixUuid));
+ throw new UnsupportedOperationException();
+
+ if (acc == null)
+ return max;
+ else
+ return Math.max(acc, max);
+
+ }, key -> {
+ if (key instanceof BlockMatrixKey)
+ return ((BlockMatrixKey)key).matrixId().equals(matrixUuid);
+ else if (key instanceof IgniteBiTuple)
+ return ((IgniteBiTuple<Integer, IgniteUuid>)key).get2().equals(matrixUuid);
+ else
+ throw new UnsupportedOperationException();
+ });
return Collections.max(maxes);
}
@@ -279,17 +342,41 @@ public class CacheUtils {
* @param matrixUuid Matrix UUID.
* @param mapper Mapping {@link IgniteFunction}.
*/
- public static <K, V> void sparseMap(IgniteUuid matrixUuid, IgniteFunction<Double, Double> mapper) {
- foreach(SparseDistributedMatrixStorage.ML_CACHE_NAME, (CacheEntry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> ce) -> {
- IgniteBiTuple k = ce.entry().getKey();
+ @SuppressWarnings("unchecked")
+ public static <K, V> void sparseMap(IgniteUuid matrixUuid, IgniteDoubleFunction<Double> mapper, String cacheName) {
+ A.notNull(matrixUuid, "matrixUuid");
+ A.notNull(cacheName, "cacheName");
+ A.notNull(mapper, "mapper");
+
+ foreach(cacheName, (CacheEntry<K, V> ce) -> {
+ K k = ce.entry().getKey();
+
+ V v = ce.entry().getValue();
- Map<Integer, Double> v = ce.entry().getValue();
+ if (v instanceof Map) {
+ Map<Integer, Double> map = (Map<Integer, Double>)v;
- for (Map.Entry<Integer, Double> e : v.entrySet())
- e.setValue(mapper.apply(e.getValue()));
+ for (Map.Entry<Integer, Double> e : (map.entrySet()))
+ e.setValue(mapper.apply(e.getValue()));
+
+ }
+ else if (v instanceof BlockEntry) {
+ BlockEntry be = (BlockEntry)v;
+
+ be.map(mapper);
+ }
+ else
+ throw new UnsupportedOperationException();
ce.cache().put(k, v);
- }, key -> key.get2().equals(matrixUuid));
+ }, key -> {
+ if (key instanceof BlockMatrixKey)
+ return ((BlockMatrixKey)key).matrixId().equals(matrixUuid);
+ else if (key instanceof IgniteBiTuple)
+ return ((IgniteBiTuple<Integer, IgniteUuid>)key).get2().equals(matrixUuid);
+ else
+ throw new UnsupportedOperationException();
+ });
}
/**
@@ -327,8 +414,7 @@ public class CacheUtils {
// Iterate over given partition.
// Query returns an empty cursor if this partition is not stored on this node.
- for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part,
- (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k)))))
+ for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part, (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k)))))
fun.accept(new CacheEntry<>(entry, cache));
}
});
@@ -387,12 +473,34 @@ public class CacheUtils {
});
}
+ /**
+ * Distributed version of fold operation.
+ *
+ * @param cacheName Cache name.
+ * @param folder Folder.
+ * @param keyFilter Key filter.
+ * @param accumulator Accumulator.
+ * @param zeroVal Zero value.
+ */
public static <K, V, A> A distributedFold(String cacheName, IgniteBiFunction<Cache.Entry<K, V>, A, A> folder,
IgnitePredicate<K> keyFilter, BinaryOperator<A> accumulator, A zeroVal) {
return sparseFold(cacheName, folder, keyFilter, accumulator, zeroVal, null, null, 0,
false);
}
+ /**
+ * Sparse version of fold. This method also applicable to sparse zeroes.
+ *
+ * @param cacheName Cache name.
+ * @param folder Folder.
+ * @param keyFilter Key filter.
+ * @param accumulator Accumulator.
+ * @param zeroVal Zero value.
+ * @param defVal Def value.
+ * @param defKey Def key.
+ * @param defValCnt Def value count.
+ * @param isNilpotent Is nilpotent.
+ */
private static <K, V, A> A sparseFold(String cacheName, IgniteBiFunction<Cache.Entry<K, V>, A, A> folder,
IgnitePredicate<K> keyFilter, BinaryOperator<A> accumulator, A zeroVal, V defVal, K defKey, long defValCnt,
boolean isNilpotent) {
@@ -411,7 +519,7 @@ public class CacheUtils {
// Use affinity in filter for ScanQuery. Otherwise we accept consumer in each node which is wrong.
Affinity affinity = ignite.affinity(cacheName);
- ClusterNode localNode = ignite.cluster().localNode();
+ ClusterNode locNode = ignite.cluster().localNode();
A a = zeroVal;
@@ -422,7 +530,7 @@ public class CacheUtils {
// Iterate over given partition.
// Query returns an empty cursor if this partition is not stored on this node.
for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part,
- (k, v) -> affinity.mapPartitionToNode(p) == localNode && (keyFilter == null || keyFilter.apply(k)))))
+ (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k)))))
a = folder.apply(entry, a);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
index d1d3904..3dc9b43 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
@@ -503,7 +503,7 @@ public abstract class AbstractMatrix implements Matrix {
/** {@inheritDoc} */
@Override public double determinant() {
- //TODO: This decomposition should be cached
+ //TODO: IGNITE-5799, This decomposition should be cached
LUDecomposition dec = new LUDecomposition(this);
double res = dec.determinant();
dec.destroy();
@@ -515,7 +515,7 @@ public abstract class AbstractMatrix implements Matrix {
if (rowSize() != columnSize())
throw new CardinalityException(rowSize(), columnSize());
- //TODO: This decomposition should be cached
+ //TODO: IGNITE-5799, This decomposition should be cached
LUDecomposition dec = new LUDecomposition(this);
Matrix res = dec.solve(likeIdentity());
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/BlockEntry.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/BlockEntry.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/BlockEntry.java
new file mode 100644
index 0000000..47f07ce
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/BlockEntry.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.impls.matrix;
+
+import org.apache.ignite.ml.math.Matrix;
+
+/**
+ * Block for {@link SparseBlockDistributedMatrix}.
+ */
+public final class BlockEntry extends SparseLocalOnHeapMatrix {
+ /** Max block size. */
+ public static final int MAX_BLOCK_SIZE = 32;
+
+ /** */
+ public BlockEntry() {
+ // No-op.
+ }
+
+ /** */
+ public BlockEntry(int row, int col) {
+ super(row, col);
+
+ assert col <= MAX_BLOCK_SIZE;
+ assert row <= MAX_BLOCK_SIZE;
+ }
+
+ /** */
+ public BlockEntry(Matrix mtx) {
+ assert mtx.columnSize() <= MAX_BLOCK_SIZE;
+ assert mtx.rowSize() <= MAX_BLOCK_SIZE;
+
+ setStorage(mtx.getStorage());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java
index a7f0afc..7f00bcb 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java
@@ -65,7 +65,6 @@ public class CacheMatrix<K, V> extends AbstractMatrix {
/**
*
- *
*/
@SuppressWarnings({"unchecked"})
private CacheMatrixStorage<K, V> storage() {
@@ -93,7 +92,7 @@ public class CacheMatrix<K, V> extends AbstractMatrix {
* @param d Value to divide to.
*/
@Override public Matrix divide(double d) {
- return mapOverValues((Double v) -> v / d);
+ return mapOverValues(v -> v / d);
}
/**
@@ -102,7 +101,7 @@ public class CacheMatrix<K, V> extends AbstractMatrix {
* @param x Value to add.
*/
@Override public Matrix plus(double x) {
- return mapOverValues((Double v) -> v + x);
+ return mapOverValues(v -> v + x);
}
/**
@@ -111,12 +110,12 @@ public class CacheMatrix<K, V> extends AbstractMatrix {
* @param x Value to multiply to.
*/
@Override public Matrix times(double x) {
- return mapOverValues((Double v) -> v * x);
+ return mapOverValues(v -> v * x);
}
/** {@inheritDoc} */
@Override public Matrix assign(double val) {
- return mapOverValues((Double v) -> val);
+ return mapOverValues(v -> val);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java
new file mode 100644
index 0000000..b3481f9
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.impls.matrix;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.StorageConstants;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.exceptions.CardinalityException;
+import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
+import org.apache.ignite.ml.math.functions.IgniteDoubleFunction;
+import org.apache.ignite.ml.math.impls.CacheUtils;
+import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixKey;
+import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixStorage;
+
+/**
+ * Sparse block distributed matrix. This matrix represented by blocks 32x32 {@link BlockEntry}.
+ *
+ * Using separate cache with keys {@link BlockMatrixKey} and values {@link BlockEntry}.
+ */
+public class SparseBlockDistributedMatrix extends AbstractMatrix implements StorageConstants {
+ /**
+ *
+ */
+ public SparseBlockDistributedMatrix() {
+ // No-op.
+ }
+
+ /**
+ * @param rows Amount of rows in the matrix.
+ * @param cols Amount of columns in the matrix.
+ */
+ public SparseBlockDistributedMatrix(int rows, int cols) {
+ assert rows > 0;
+ assert cols > 0;
+
+ setStorage(new BlockMatrixStorage(rows, cols));
+ }
+
+ /**
+ * Return the same matrix with updates values (broken contract).
+ *
+ * @param d Value to divide to.
+ */
+ @Override public Matrix divide(double d) {
+ return mapOverValues(v -> v / d);
+ }
+
+ /**
+ * Return the same matrix with updates values (broken contract).
+ *
+ * @param x Value to add.
+ */
+ @Override public Matrix plus(double x) {
+ return mapOverValues(v -> v + x);
+ }
+
+ /**
+ * Return the same matrix with updates values (broken contract).
+ *
+ * @param x Value to multiply.
+ */
+ @Override public Matrix times(double x) {
+ return mapOverValues(v -> v * x);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @SuppressWarnings({"unchecked"})
+ @Override public Matrix times(final Matrix mtx) {
+ if (mtx == null)
+ throw new IllegalArgumentException("The matrix should be not null.");
+
+ if (columnSize() != mtx.rowSize())
+ throw new CardinalityException(columnSize(), mtx.rowSize());
+
+ SparseBlockDistributedMatrix matrixA = this;
+ SparseBlockDistributedMatrix matrixB = (SparseBlockDistributedMatrix)mtx;
+
+ String cacheName = BlockMatrixStorage.ML_BLOCK_CACHE_NAME;
+ SparseBlockDistributedMatrix matrixC = new SparseBlockDistributedMatrix(matrixA.rowSize(), matrixB.columnSize());
+
+ CacheUtils.bcast(BlockMatrixStorage.ML_BLOCK_CACHE_NAME, () -> {
+ Ignite ignite = Ignition.localIgnite();
+ Affinity affinity = ignite.affinity(cacheName);
+
+ IgniteCache<BlockMatrixKey, BlockEntry> cache = ignite.getOrCreateCache(cacheName);
+ ClusterNode locNode = ignite.cluster().localNode();
+
+ BlockMatrixStorage storageC = matrixC.storage();
+
+ Map<ClusterNode, Collection<BlockMatrixKey>> keysCToNodes = affinity.mapKeysToNodes(storageC.getAllKeys());
+ Collection<BlockMatrixKey> locKeys = keysCToNodes.get(locNode);
+
+ if (locKeys == null)
+ return;
+
+ // compute Cij locally on each node
+ // TODO: IGNITE:5114, exec in parallel
+ locKeys.forEach(key -> {
+ long newBlockId = key.blockId();
+ BlockEntry blockC = null;
+
+ List<BlockEntry> aRow = matrixA.storage().getRowForBlock(newBlockId, storageC);
+ List<BlockEntry> bCol = matrixB.storage().getColForBlock(newBlockId, storageC);
+
+ for (int i = 0; i < aRow.size(); i++) {
+ BlockEntry blockA = aRow.get(i);
+ BlockEntry blockB = bCol.get(i);
+
+ BlockEntry tmpBlock = new BlockEntry(blockA.times(blockB));
+
+ blockC = blockC == null ? tmpBlock : new BlockEntry(blockC.plus(tmpBlock));
+ }
+
+ cache.put(storageC.getCacheKey(newBlockId), blockC);
+ });
+ });
+
+ return matrixC;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix assign(double val) {
+ return mapOverValues(v -> val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix map(IgniteDoubleFunction<Double> fun) {
+ return mapOverValues(fun);
+ }
+
+ /** {@inheritDoc} */
+ @Override public double sum() {
+ return CacheUtils.sparseSum(getUUID(), BlockMatrixStorage.ML_BLOCK_CACHE_NAME);
+ }
+
+ /** {@inheritDoc} */
+ @Override public double maxValue() {
+ return CacheUtils.sparseMax(getUUID(), BlockMatrixStorage.ML_BLOCK_CACHE_NAME);
+ }
+
+ /** {@inheritDoc} */
+ @Override public double minValue() {
+ return CacheUtils.sparseMin(getUUID(), BlockMatrixStorage.ML_BLOCK_CACHE_NAME);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix copy() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix like(int rows, int cols) {
+ return new SparseBlockDistributedMatrix(rows, cols);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Vector likeVector(int crd) {
+ throw new UnsupportedOperationException();
+ }
+
+ /** */
+ private IgniteUuid getUUID() {
+ return ((BlockMatrixStorage)getStorage()).getUUID();
+ }
+
+ /**
+ * @param mapper Mapping function.
+ * @return Matrix with mapped values.
+ */
+ private Matrix mapOverValues(IgniteDoubleFunction<Double> mapper) {
+ CacheUtils.sparseMap(getUUID(), mapper, BlockMatrixStorage.ML_BLOCK_CACHE_NAME);
+
+ return this;
+ }
+
+ /**
+ *
+ */
+ private BlockMatrixStorage storage() {
+ return (BlockMatrixStorage)getStorage();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
index df2ddc4..a86db95 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
@@ -23,7 +23,6 @@ import org.apache.ignite.ml.math.StorageConstants;
import org.apache.ignite.ml.math.Vector;
import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
import org.apache.ignite.ml.math.functions.IgniteDoubleFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.ml.math.impls.CacheUtils;
import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
@@ -61,10 +60,7 @@ public class SparseDistributedMatrix extends AbstractMatrix implements StorageCo
setStorage(new SparseDistributedMatrixStorage(rows, cols, stoMode, acsMode));
}
- /**
- *
- *
- */
+ /** */
private SparseDistributedMatrixStorage storage() {
return (SparseDistributedMatrixStorage)getStorage();
}
@@ -75,7 +71,7 @@ public class SparseDistributedMatrix extends AbstractMatrix implements StorageCo
* @param d Value to divide to.
*/
@Override public Matrix divide(double d) {
- return mapOverValues((Double v) -> v / d);
+ return mapOverValues(v -> v / d);
}
/**
@@ -84,7 +80,7 @@ public class SparseDistributedMatrix extends AbstractMatrix implements StorageCo
* @param x Value to add.
*/
@Override public Matrix plus(double x) {
- return mapOverValues((Double v) -> v + x);
+ return mapOverValues(v -> v + x);
}
/**
@@ -93,42 +89,42 @@ public class SparseDistributedMatrix extends AbstractMatrix implements StorageCo
* @param x Value to multiply.
*/
@Override public Matrix times(double x) {
- return mapOverValues((Double v) -> v * x);
+ return mapOverValues(v -> v * x);
}
/** {@inheritDoc} */
@Override public Matrix assign(double val) {
- return mapOverValues((Double v) -> val);
+ return mapOverValues(v -> val);
}
/** {@inheritDoc} */
@Override public Matrix map(IgniteDoubleFunction<Double> fun) {
- return mapOverValues(fun::apply);
+ return mapOverValues(fun);
}
/**
* @param mapper Mapping function.
* @return Matrix with mapped values.
*/
- private Matrix mapOverValues(IgniteFunction<Double, Double> mapper) {
- CacheUtils.sparseMap(getUUID(), mapper);
+ private Matrix mapOverValues(IgniteDoubleFunction<Double> mapper) {
+ CacheUtils.sparseMap(getUUID(), mapper, SparseDistributedMatrixStorage.ML_CACHE_NAME);
return this;
}
/** {@inheritDoc} */
@Override public double sum() {
- return CacheUtils.sparseSum(getUUID());
+ return CacheUtils.sparseSum(getUUID(), SparseDistributedMatrixStorage.ML_CACHE_NAME);
}
/** {@inheritDoc} */
@Override public double maxValue() {
- return CacheUtils.sparseMax(getUUID());
+ return CacheUtils.sparseMax(getUUID(), SparseDistributedMatrixStorage.ML_CACHE_NAME);
}
/** {@inheritDoc} */
@Override public double minValue() {
- return CacheUtils.sparseMin(getUUID());
+ return CacheUtils.sparseMin(getUUID(), SparseDistributedMatrixStorage.ML_CACHE_NAME);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java
new file mode 100644
index 0000000..74ddfe5
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.impls.storage.matrix;
+
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix;
+
+/**
+ * Cache key for blocks in {@link SparseBlockDistributedMatrix}.
+ */
+public interface BaseBlockMatrixKey {
+ /**
+ * @return block id.
+ */
+ public long blockId();
+
+ /**
+ * @return matrix id.
+ */
+ public IgniteUuid matrixId();
+
+ /**
+ * @return key affinity key.
+ */
+ public IgniteUuid affinityKey();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java
new file mode 100644
index 0000000..3749f44
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.impls.storage.matrix;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.binary.Binarylizable;
+import org.apache.ignite.internal.binary.BinaryUtils;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.ml.math.impls.matrix.BlockEntry;
+import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Key implementation for {@link BlockEntry} using for {@link SparseBlockDistributedMatrix}.
+ */
+public class BlockMatrixKey implements BaseBlockMatrixKey, Externalizable, Binarylizable {
+ /** */
+ private static final long serialVersionUID = 0L;
+ /** Block ID */
+ private long blockId;
+ /** Matrix ID */
+ private IgniteUuid matrixUuid;
+ /** Block affinity key. */
+ private IgniteUuid affinityKey;
+
+ /**
+ * Empty constructor required for {@link Externalizable}.
+ */
+ public BlockMatrixKey() {
+ // No-op.
+ }
+
+ /**
+ * Construct matrix block key.
+ *
+ * @param blockId Block id.
+ * @param matrixUuid Matrix uuid.
+ * @param affinityKey Affinity key.
+ */
+ public BlockMatrixKey(long blockId, IgniteUuid matrixUuid, @Nullable IgniteUuid affinityKey) {
+ assert blockId >= 0;
+ assert matrixUuid != null;
+
+ this.blockId = blockId;
+ this.matrixUuid = matrixUuid;
+ this.affinityKey = affinityKey;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long blockId() {
+ return blockId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid matrixId() {
+ return matrixUuid;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid affinityKey() {
+ return affinityKey;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ U.writeGridUuid(out, matrixUuid);
+ U.writeGridUuid(out, affinityKey);
+ out.writeLong(blockId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ matrixUuid = U.readGridUuid(in);
+ affinityKey = U.readGridUuid(in);
+ blockId = in.readLong();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
+ BinaryRawWriter out = writer.rawWriter();
+
+ BinaryUtils.writeIgniteUuid(out, matrixUuid);
+ BinaryUtils.writeIgniteUuid(out, affinityKey);
+ out.writeLong(blockId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
+ BinaryRawReader in = reader.rawReader();
+
+ matrixUuid = BinaryUtils.readIgniteUuid(in);
+ affinityKey = BinaryUtils.readIgniteUuid(in);
+ blockId = in.readLong();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return matrixUuid.hashCode() + (int)(blockId ^ (blockId >>> 32));
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+
+ if (obj == null || obj.getClass() != getClass())
+ return false;
+
+ BlockMatrixKey that = (BlockMatrixKey)obj;
+
+ return blockId == that.blockId && matrixUuid.equals(that.matrixUuid) && F.eq(affinityKey, that.affinityKey);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(BlockMatrixKey.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java
new file mode 100644
index 0000000..6640e5a
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java
@@ -0,0 +1,435 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.impls.storage.matrix;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.LongStream;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.ml.math.MatrixStorage;
+import org.apache.ignite.ml.math.StorageConstants;
+import org.apache.ignite.ml.math.impls.CacheUtils;
+import org.apache.ignite.ml.math.impls.matrix.BlockEntry;
+import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix;
+
+import static org.apache.ignite.ml.math.impls.matrix.BlockEntry.MAX_BLOCK_SIZE;
+
+/**
+ * Storage for {@link SparseBlockDistributedMatrix}.
+ */
+public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, StorageConstants {
+ /** Cache name used for all instances of {@link BlockMatrixStorage}. */
+ public static final String ML_BLOCK_CACHE_NAME = "ML_BLOCK_SPARSE_MATRICES_CONTAINER";
+ /** */
+ private int blocksInCol;
+ /** */
+ private int blocksInRow;
+ /** Amount of rows in the matrix. */
+ private int rows;
+ /** Amount of columns in the matrix. */
+ private int cols;
+ /** Matrix uuid. */
+ private IgniteUuid uuid;
+ /** Block size about 8 KB of data. */
+ private int maxBlockEdge = MAX_BLOCK_SIZE;
+
+ /** Actual distributed storage. */
+ private IgniteCache<
+ BlockMatrixKey /* Matrix block number with uuid. */,
+ BlockEntry /* Block of matrix, local sparse matrix. */
+ > cache = null;
+
+ /**
+ *
+ */
+ public BlockMatrixStorage() {
+ // No-op.
+ }
+
+ /**
+ * @param rows Amount of rows in the matrix.
+ * @param cols Amount of columns in the matrix.
+ */
+ public BlockMatrixStorage(int rows, int cols) {
+ assert rows > 0;
+ assert cols > 0;
+
+ this.rows = rows;
+ this.cols = cols;
+
+ //cols % maxBlockEdge > 0 ? 1 : 0
+
+ this.blocksInRow = cols % maxBlockEdge == 0 ? cols / maxBlockEdge : cols / maxBlockEdge + 1;
+ this.blocksInCol = rows % maxBlockEdge == 0 ? rows / maxBlockEdge : rows / maxBlockEdge + 1;
+
+ cache = newCache();
+
+ uuid = IgniteUuid.randomUuid();
+ }
+
+ /**
+ *
+ */
+ public IgniteCache<BlockMatrixKey, BlockEntry> cache() {
+ return cache;
+ }
+
+ /** {@inheritDoc} */
+ @Override public double get(int x, int y) {
+ return matrixGet(x, y);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void set(int x, int y, double v) {
+ matrixSet(x, y, v);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int columnSize() {
+ return cols;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int rowSize() {
+ return rows;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeInt(rows);
+ out.writeInt(cols);
+ out.writeInt(blocksInRow);
+ out.writeInt(blocksInCol);
+ U.writeGridUuid(out, uuid);
+ out.writeUTF(cache.getName());
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ rows = in.readInt();
+ cols = in.readInt();
+ blocksInRow = in.readInt();
+ blocksInCol = in.readInt();
+ uuid = U.readGridUuid(in);
+ cache = ignite().getOrCreateCache(in.readUTF());
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isSequentialAccess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isDense() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isRandomAccess() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isDistributed() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isArrayBased() {
+ return false;
+ }
+
+ /** Delete all data from cache. */
+ @Override public void destroy() {
+ long maxBlockId = getBlockId(cols, rows);
+
+ Set<BlockMatrixKey> keyset = LongStream.rangeClosed(0, maxBlockId).mapToObj(this::getCacheKey).collect(Collectors.toSet());
+
+ cache.clearAll(keyset);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = 1;
+
+ res = res * 37 + cols;
+ res = res * 37 + rows;
+ res = res * 37 + uuid.hashCode();
+ res = res * 37 + cache.hashCode();
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+
+ if (obj == null || getClass() != obj.getClass())
+ return false;
+
+ BlockMatrixStorage that = (BlockMatrixStorage)obj;
+
+ return rows == that.rows && cols == that.cols && uuid.equals(that.uuid)
+ && (cache != null ? cache.equals(that.cache) : that.cache == null);
+ }
+
+ /**
+ * Get storage UUID.
+ *
+ * @return storage UUID.
+ */
+ public IgniteUuid getUUID() {
+ return uuid;
+ }
+
+ /**
+ * Build the cache key for the given block id
+ */
+ public BlockMatrixKey getCacheKey(long blockId) {
+ return new BlockMatrixKey(blockId, uuid, getAffinityKey(blockId));
+ }
+
+ /**
+ * Get rows for current block.
+ *
+ * @param blockId block id.
+ * @param storageC result storage.
+ * @return The list of block entries.
+ */
+ public List<BlockEntry> getRowForBlock(long blockId, BlockMatrixStorage storageC) {
+ long blockRow = blockId / storageC.blocksInCol;
+ long blockCol = blockId % storageC.blocksInRow;
+
+ long locBlock = this.blocksInRow * (blockRow) + (blockCol >= this.blocksInRow ? (blocksInRow - 1) : blockCol);
+
+ return getRowForBlock(locBlock);
+ }
+
+ /**
+ * Get cols for current block.
+ *
+ * @param blockId block id.
+ * @param storageC result storage.
+ * @return The list of block entries.
+ */
+ public List<BlockEntry> getColForBlock(long blockId, BlockMatrixStorage storageC) {
+ long blockRow = blockId / storageC.blocksInCol;
+ long blockCol = blockId % storageC.blocksInRow;
+
+ long locBlock = this.blocksInRow * (blockRow) + (blockCol >= this.blocksInRow ? (blocksInRow - 1) : blockCol);
+
+ return getColForBlock(locBlock);
+ }
+
+ /**
+ * Build a keyset for this matrix storage.
+ */
+ public Collection<BlockMatrixKey> getAllKeys() {
+ long maxBlockId = numberOfBlocks();
+ Collection<BlockMatrixKey> keys = new LinkedList<>();
+
+ for (long id = 0; id < maxBlockId; id++)
+ keys.add(getCacheKey(id));
+
+ return keys;
+ }
+
+ /** */
+ private List<BlockEntry> getRowForBlock(long blockId) {
+ List<BlockEntry> res = new LinkedList<>();
+
+ boolean isFirstRow = blockId < blocksInRow;
+
+ long startBlock = isFirstRow ? 0 : blockId - blockId % blocksInRow;
+ long endBlock = startBlock + blocksInRow - 1;
+
+ for (long i = startBlock; i <= endBlock; i++)
+ res.add(getEntryById(i));
+
+ return res;
+ }
+
+ /** */
+ private List<BlockEntry> getColForBlock(long blockId) {
+ List<BlockEntry> res = new LinkedList<>();
+
+ long startBlock = blockId % blocksInRow;
+ long endBlock = startBlock + blocksInRow * (blocksInCol - 1);
+
+ for (long i = startBlock; i <= endBlock; i += blocksInRow)
+ res.add(getEntryById(i));
+
+ return res;
+ }
+
+ /**
+ *
+ */
+ private BlockEntry getEntryById(long blockId) {
+ BlockMatrixKey key = getCacheKey(blockId);
+
+ BlockEntry entry = cache.localPeek(key);
+ entry = entry != null ? entry : cache.get(key);
+
+ if (entry == null) {
+ long colId = blockId == 0 ? 0 : blockId + 1;
+
+ boolean isLastRow = (blockId) >= blocksInRow * (blocksInCol - 1);
+ boolean isLastCol = (colId) % blocksInRow == 0;
+
+ entry = new BlockEntry(isLastRow && rows % maxBlockEdge != 0 ? rows % maxBlockEdge : maxBlockEdge, isLastCol && cols % maxBlockEdge != 0 ? cols % maxBlockEdge : maxBlockEdge);
+ }
+
+ return entry;
+ }
+
+ /**
+ *
+ */
+ private long numberOfBlocks() {
+ int rows = rowSize();
+ int cols = columnSize();
+
+ return ((rows / maxBlockEdge) + (((rows % maxBlockEdge) > 0) ? 1 : 0))
+ * ((cols / maxBlockEdge) + (((cols % maxBlockEdge) > 0) ? 1 : 0));
+ }
+
+ /**
+ * TODO: IGNITE-5646, WIP
+ *
+ * Get affinity key for the given id.
+ */
+ private IgniteUuid getAffinityKey(long id) {
+ return null;
+ }
+
+ /**
+ * Distributed matrix set.
+ *
+ * @param a Row or column index.
+ * @param b Row or column index.
+ * @param v New value to set.
+ */
+ private void matrixSet(int a, int b, double v) {
+ long id = getBlockId(a, b);
+ // Remote set on the primary node (where given row or column is stored locally).
+ ignite().compute(groupForKey(ML_BLOCK_CACHE_NAME, id)).run(() -> {
+ IgniteCache<BlockMatrixKey, BlockEntry> cache = Ignition.localIgnite().getOrCreateCache(ML_BLOCK_CACHE_NAME);
+
+ BlockMatrixKey key = getCacheKey(getBlockId(a, b));
+
+ // Local get.
+ BlockEntry block = cache.localPeek(key, CachePeekMode.PRIMARY);
+
+ if (block == null)
+ block = cache.get(key); //Remote entry get.
+
+ if (block == null)
+ block = initBlockFor(a, b);
+
+ block.set(a % block.rowSize(), b % block.columnSize(), v);
+
+ // Local put.
+ cache.put(key, block);
+ });
+ }
+
+ /** */
+ private long getBlockId(int x, int y) {
+ return (y / maxBlockEdge) * blockShift(cols) + (x / maxBlockEdge);
+ }
+
+ /** */
+ private BlockEntry initBlockFor(int x, int y) {
+ int blockRows = rows - x >= maxBlockEdge ? maxBlockEdge : rows - x;
+ int blockCols = cols - y >= maxBlockEdge ? maxBlockEdge : cols - y;
+
+ return new BlockEntry(blockRows, blockCols);
+ }
+
+ /** */
+ private int blockShift(int i) {
+ return (i) / maxBlockEdge + ((i) % maxBlockEdge > 0 ? 1 : 0);
+ }
+
+ /**
+ * Distributed matrix get.
+ *
+ * @param a Row or column index.
+ * @param b Row or column index.
+ * @return Matrix value at (a, b) index.
+ */
+ private double matrixGet(int a, int b) {
+ // Remote get from the primary node (where given row or column is stored locally).
+ return ignite().compute(groupForKey(ML_BLOCK_CACHE_NAME, getBlockId(a, b))).call(() -> {
+ IgniteCache<BlockMatrixKey, BlockEntry> cache = Ignition.localIgnite().getOrCreateCache(ML_BLOCK_CACHE_NAME);
+
+ BlockMatrixKey key = getCacheKey(getBlockId(a, b));
+
+ // Local get.
+ BlockEntry block = cache.localPeek(key, CachePeekMode.PRIMARY);
+
+ if (block == null)
+ block = cache.get(key);
+
+ return block == null ? 0.0 : block.get(a % block.rowSize(), b % block.columnSize());
+ });
+ }
+
+ /**
+ * Create new ML cache if needed.
+ */
+ private IgniteCache<BlockMatrixKey, BlockEntry> newCache() {
+ CacheConfiguration<BlockMatrixKey, BlockEntry> cfg = new CacheConfiguration<>();
+
+ // Write to primary.
+ cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
+
+ // Atomic transactions only.
+ cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+ // No eviction.
+ cfg.setEvictionPolicy(null);
+
+ // No copying of values.
+ cfg.setCopyOnRead(false);
+
+ // Cache is partitioned.
+ cfg.setCacheMode(CacheMode.PARTITIONED);
+
+ // Random cache name.
+ cfg.setName(ML_BLOCK_CACHE_NAME);
+
+ return Ignition.localIgnite().getOrCreateCache(cfg);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java
index f2efe74..5145376 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java
@@ -46,9 +46,7 @@ public class SparseLocalOnHeapVectorStorage implements VectorStorage, StorageCon
// No-op.
}
- /**
- * @param map
- */
+ /** */
public SparseLocalOnHeapVectorStorage(Map<Integer, Double> map, boolean copy) {
assert map.size() > 0;
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java
index e406b5b..525e6e9 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java
@@ -30,6 +30,7 @@ public class Variance {
/** */
private double m2;
+ /** */
public Variance() {
mean = 0;
n = 0;
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/package-info.java
new file mode 100644
index 0000000..7b65fce
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Statistics stuff.
+ */
+package org.apache.ignite.ml.math.statistics;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java
index 6c25f0e..9190901 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java
@@ -25,7 +25,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
- *
+ * Some {@link Map} related utils.
*/
public class MapUtil {
/** */
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/util/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/package-info.java
new file mode 100644
index 0000000..2507ee4
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Some math utils.
+ */
+package org.apache.ignite.ml.math.util;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/package-info.java
new file mode 100644
index 0000000..779581b
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Root ML package.
+ */
+package org.apache.ignite.ml;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java
index 9899d3b..5dc860c 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java
@@ -18,6 +18,7 @@
package org.apache.ignite.ml.math;
import org.apache.ignite.ml.math.impls.matrix.CacheMatrixTest;
+import org.apache.ignite.ml.math.impls.matrix.SparseDistributedBlockMatrixTest;
import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrixTest;
import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorageTest;
import org.apache.ignite.ml.math.impls.vector.CacheVectorTest;
@@ -33,6 +34,7 @@ import org.junit.runners.Suite;
CacheMatrixTest.class,
SparseDistributedMatrixStorageTest.class,
SparseDistributedMatrixTest.class,
+ SparseDistributedBlockMatrixTest.class
})
public class MathImplDistributedTestSuite {
// No-op.
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java
new file mode 100644
index 0000000..1228f05
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.impls.matrix;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
+import org.apache.ignite.ml.math.impls.MathTestConstants;
+import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixKey;
+import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixStorage;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testframework.junits.common.GridCommonTest;
+
+import static org.apache.ignite.ml.math.impls.MathTestConstants.UNEXPECTED_VAL;
+
+/**
+ * Tests for {@link SparseBlockDistributedMatrix}.
+ */
+@GridCommonTest(group = "Distributed Models")
+public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest {
+ /** Number of nodes in grid */
+ private static final int NODE_COUNT = 3;
+ /** Precision. */
+ private static final double PRECISION = 0.0;
+ /** Grid instance. */
+ private Ignite ignite;
+ /** Matrix rows */
+ private final int rows = MathTestConstants.STORAGE_SIZE;
+ /** Matrix cols */
+ private final int cols = MathTestConstants.STORAGE_SIZE;
+ /** Matrix for tests */
+ private SparseBlockDistributedMatrix cacheMatrix;
+
+ /**
+ * Default constructor.
+ */
+ public SparseDistributedBlockMatrixTest() {
+ super(false);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ for (int i = 1; i <= NODE_COUNT; i++)
+ startGrid(i);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void beforeTest() throws Exception {
+ ignite = grid(NODE_COUNT);
+
+ ignite.configuration().setPeerClassLoadingEnabled(true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ if (cacheMatrix != null) {
+ cacheMatrix.destroy();
+ cacheMatrix = null;
+ }
+ }
+
+ /** */
+ public void testGetSet() throws Exception {
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ cacheMatrix = new SparseBlockDistributedMatrix(rows, cols);
+
+ for (int i = 0; i < rows; i++) {
+ for (int j = 0; j < cols; j++) {
+ double v = Math.random();
+ cacheMatrix.set(i, j, v);
+
+ assertEquals("Unexpected value for matrix element["+ i +" " + j + "]", v, cacheMatrix.get(i, j), PRECISION);
+ }
+ }
+ }
+
+ /** */
+ public void testExternalize() throws IOException, ClassNotFoundException {
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ cacheMatrix = new SparseBlockDistributedMatrix(rows, cols);
+
+ cacheMatrix.set(1, 1, 1.0);
+
+ ByteArrayOutputStream byteArrOutputStream = new ByteArrayOutputStream();
+ ObjectOutputStream objOutputStream = new ObjectOutputStream(byteArrOutputStream);
+
+ objOutputStream.writeObject(cacheMatrix);
+
+ ByteArrayInputStream byteArrInputStream = new ByteArrayInputStream(byteArrOutputStream.toByteArray());
+ ObjectInputStream objInputStream = new ObjectInputStream(byteArrInputStream);
+
+ SparseBlockDistributedMatrix objRestored = (SparseBlockDistributedMatrix)objInputStream.readObject();
+
+ assertTrue(MathTestConstants.VAL_NOT_EQUALS, cacheMatrix.equals(objRestored));
+ assertEquals(MathTestConstants.VAL_NOT_EQUALS, objRestored.get(1, 1), 1.0, PRECISION);
+ }
+
+ /** Test simple math. */
+ public void testMath() {
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ cacheMatrix = new SparseBlockDistributedMatrix(rows, cols);
+ initMtx(cacheMatrix);
+
+ cacheMatrix.assign(2.0);
+ for (int i = 0; i < cacheMatrix.rowSize(); i++)
+ for (int j = 0; j < cacheMatrix.columnSize(); j++)
+ assertEquals(UNEXPECTED_VAL, 2.0, cacheMatrix.get(i, j), PRECISION);
+
+ cacheMatrix.plus(3.0);
+ for (int i = 0; i < cacheMatrix.rowSize(); i++)
+ for (int j = 0; j < cacheMatrix.columnSize(); j++)
+ assertEquals(UNEXPECTED_VAL, 5.0, cacheMatrix.get(i, j), PRECISION);
+
+ cacheMatrix.times(2.0);
+ for (int i = 0; i < cacheMatrix.rowSize(); i++)
+ for (int j = 0; j < cacheMatrix.columnSize(); j++)
+ assertEquals(UNEXPECTED_VAL, 10.0, cacheMatrix.get(i, j), PRECISION);
+
+ cacheMatrix.divide(10.0);
+ for (int i = 0; i < cacheMatrix.rowSize(); i++)
+ for (int j = 0; j < cacheMatrix.columnSize(); j++)
+ assertEquals(UNEXPECTED_VAL, 1.0, cacheMatrix.get(i, j), PRECISION);
+
+ assertEquals(UNEXPECTED_VAL, cacheMatrix.rowSize() * cacheMatrix.columnSize(), cacheMatrix.sum(), PRECISION);
+ }
+
+ /** */
+ public void testMinMax() {
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ cacheMatrix = new SparseBlockDistributedMatrix(rows, cols);
+
+ for (int i = 0; i < cacheMatrix.rowSize(); i++)
+ for (int j = 0; j < cacheMatrix.columnSize(); j++)
+ cacheMatrix.set(i, j, i * cols + j + 1);
+
+ assertEquals(UNEXPECTED_VAL, 1.0, cacheMatrix.minValue(), PRECISION);
+ assertEquals(UNEXPECTED_VAL, rows * cols, cacheMatrix.maxValue(), PRECISION);
+
+ for (int i = 0; i < cacheMatrix.rowSize(); i++)
+ for (int j = 0; j < cacheMatrix.columnSize(); j++)
+ cacheMatrix.set(i, j, -1.0 * (i * cols + j + 1));
+
+ assertEquals(UNEXPECTED_VAL, -rows * cols, cacheMatrix.minValue(), PRECISION);
+ assertEquals(UNEXPECTED_VAL, -1.0, cacheMatrix.maxValue(), PRECISION);
+
+ for (int i = 0; i < cacheMatrix.rowSize(); i++)
+ for (int j = 0; j < cacheMatrix.columnSize(); j++)
+ cacheMatrix.set(i, j, i * cols + j);
+
+ assertEquals(UNEXPECTED_VAL, 0.0, cacheMatrix.minValue(), PRECISION);
+ assertEquals(UNEXPECTED_VAL, rows * cols - 1.0, cacheMatrix.maxValue(), PRECISION);
+ }
+
+ /** */
+ public void testMap() {
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ cacheMatrix = new SparseBlockDistributedMatrix(rows, cols);
+ initMtx(cacheMatrix);
+
+ cacheMatrix.map(i -> 100.0);
+ for (int i = 0; i < cacheMatrix.rowSize(); i++)
+ for (int j = 0; j < cacheMatrix.columnSize(); j++)
+ assertEquals(UNEXPECTED_VAL, 100.0, cacheMatrix.get(i, j), PRECISION);
+ }
+
+ /** */
+ public void testCopy() {
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ cacheMatrix = new SparseBlockDistributedMatrix(rows, cols);
+
+ try {
+ cacheMatrix.copy();
+ fail("UnsupportedOperationException expected.");
+ }
+ catch (UnsupportedOperationException e) {
+ return;
+ }
+ fail("UnsupportedOperationException expected.");
+ }
+
+ /** */
+ public void testCacheBehaviour(){
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ SparseBlockDistributedMatrix cacheMatrix1 = new SparseBlockDistributedMatrix(rows, cols);
+ SparseBlockDistributedMatrix cacheMatrix2 = new SparseBlockDistributedMatrix(rows, cols);
+
+ initMtx(cacheMatrix1);
+ initMtx(cacheMatrix2);
+
+ Collection<String> cacheNames = ignite.cacheNames();
+
+ assert cacheNames.contains(BlockMatrixStorage.ML_BLOCK_CACHE_NAME);
+
+ IgniteCache<BlockMatrixKey, Object> cache = ignite.getOrCreateCache(BlockMatrixStorage.ML_BLOCK_CACHE_NAME);
+
+ Set<BlockMatrixKey> keySet1 = buildKeySet(cacheMatrix1);
+ Set<BlockMatrixKey> keySet2 = buildKeySet(cacheMatrix2);
+
+ assert cache.containsKeys(keySet1);
+ assert cache.containsKeys(keySet2);
+
+ cacheMatrix2.destroy();
+
+ assert cache.containsKeys(keySet1);
+ assert !cache.containsKeys(keySet2);
+
+ cacheMatrix1.destroy();
+
+ assert !cache.containsKeys(keySet1);
+ }
+
+ /** */
+ public void testLike() {
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ cacheMatrix = new SparseBlockDistributedMatrix(rows, cols);
+
+ assertNotNull(cacheMatrix.like(1, 1));
+ }
+
+ /** */
+ public void testLikeVector() {
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ cacheMatrix = new SparseBlockDistributedMatrix(rows, cols);
+
+ try {
+ cacheMatrix.likeVector(1);
+ fail("UnsupportedOperationException expected.");
+ }
+ catch (UnsupportedOperationException e) {
+ return;
+ }
+ fail("UnsupportedOperationException expected.");
+ }
+
+ /**
+ * Simple test for two square matrices.
+ */
+ public void testSquareMatrixTimes(){
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ int size = 100;
+
+ Matrix cacheMatrix1 = new SparseBlockDistributedMatrix(size, size);
+ Matrix cacheMatrix2 = new SparseBlockDistributedMatrix(size, size);
+
+ for (int i = 0; i < size; i++) {
+ cacheMatrix1.setX(i, i, i);
+ cacheMatrix2.setX(i, i, i);
+ }
+
+ Matrix res = cacheMatrix1.times(cacheMatrix2);
+
+ for(int i = 0; i < size; i++)
+ for(int j = 0; j < size; j++)
+ if (i == j)
+ assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, i * i, res.get(i, j), PRECISION);
+ else
+ assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, 0, res.get(i, j), PRECISION);
+ }
+
+ /**
+ *
+ */
+ public void testNonSquareMatrixTimes(){
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ int size = BlockEntry.MAX_BLOCK_SIZE + 1;
+ int size2 = BlockEntry.MAX_BLOCK_SIZE * 2 + 1;
+
+ Matrix cacheMatrix1 = new SparseBlockDistributedMatrix(size2, size);
+ Matrix cacheMatrix2 = new SparseBlockDistributedMatrix(size, size2);
+
+ for (int i = 0; i < size; i++) {
+ cacheMatrix1.setX(i, i, i);
+ cacheMatrix2.setX(i, i, i);
+ }
+
+ Matrix res = cacheMatrix1.times(cacheMatrix2);
+
+ for(int i = 0; i < size; i++)
+ for(int j = 0; j < size; j++)
+ if (i == j)
+ assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, i * i, res.get(i, j), PRECISION);
+ else
+ assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, 0, res.get(i, j), PRECISION);
+ }
+
+ /**
+ *
+ */
+ public void testNonSquareMatrixTimes2(){
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ int size = BlockEntry.MAX_BLOCK_SIZE + 1;
+ int size2 = BlockEntry.MAX_BLOCK_SIZE * 2 + 1;
+
+ Matrix cacheMatrix1 = new SparseBlockDistributedMatrix(size, size2);
+ Matrix cacheMatrix2 = new SparseBlockDistributedMatrix(size2, size);
+
+ for (int i = 0; i < size; i++) {
+ cacheMatrix1.setX(i, i, i);
+ cacheMatrix2.setX(i, i, i);
+ }
+
+ Matrix res = cacheMatrix1.times(cacheMatrix2);
+
+ for(int i = 0; i < size; i++)
+ for(int j = 0; j < size; j++)
+ if (i == j)
+ assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, i * i, res.get(i, j), PRECISION);
+ else
+ assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, 0, res.get(i, j), PRECISION);
+ }
+
+ /** */
+ private void initMtx(Matrix m) {
+ for (int i = 0; i < m.rowSize(); i++)
+ for (int j = 0; j < m.columnSize(); j++)
+ m.set(i, j, 1.0);
+ }
+
+ /** Build key set for SparseBlockDistributedMatrix. */
+ private Set<BlockMatrixKey> buildKeySet(SparseBlockDistributedMatrix m){
+ Set<BlockMatrixKey> set = new HashSet<>();
+
+ BlockMatrixStorage storage = (BlockMatrixStorage)m.getStorage();
+
+ IgniteUuid uuid = storage.getUUID();
+
+ long maxBlock = (rows / 32 + (rows % 32 > 0 ? 1 : 0)) * (cols / 32 + (cols % 32 > 0 ? 1 : 0));
+
+ for (long i = 0; i < maxBlock; i++)
+ set.add(new BlockMatrixKey(i,uuid,null));
+
+ return set;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java
index a7cd6b5..3fec83c 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java
@@ -48,10 +48,10 @@ import static org.apache.ignite.ml.math.impls.MathTestConstants.UNEXPECTED_VAL;
public class SparseDistributedMatrixTest extends GridCommonAbstractTest {
/** Number of nodes in grid */
private static final int NODE_COUNT = 3;
- /** Cache name. */
- private static final String CACHE_NAME = "test-cache";
/** Precision. */
private static final double PRECISION = 0.0;
+ /** */
+ private static final int MATRIX_SIZE = 10;
/** Grid instance. */
private Ignite ignite;
/** Matrix rows */
@@ -90,8 +90,6 @@ public class SparseDistributedMatrixTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
- ignite.destroyCache(CACHE_NAME);
-
if (cacheMatrix != null) {
cacheMatrix.destroy();
cacheMatrix = null;
@@ -166,7 +164,9 @@ public class SparseDistributedMatrixTest extends GridCommonAbstractTest {
assertEquals(UNEXPECTED_VAL, cacheMatrix.rowSize() * cacheMatrix.columnSize(), cacheMatrix.sum(), PRECISION);
}
- /** */
+ /**
+ * TODO: IGNITE-5102, wrong min/max, wait for fold/map fix
+ */
public void testMinMax() {
IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
@@ -286,6 +286,28 @@ public class SparseDistributedMatrixTest extends GridCommonAbstractTest {
}
/** */
+ public void testMatrixTimes(){
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ SparseDistributedMatrix cacheMatrix1 = new SparseDistributedMatrix(MATRIX_SIZE, MATRIX_SIZE, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
+ SparseDistributedMatrix cacheMatrix2 = new SparseDistributedMatrix(MATRIX_SIZE, MATRIX_SIZE, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
+
+ for (int i = 0; i < MATRIX_SIZE; i++) {
+ cacheMatrix1.setX(i, i, i);
+ cacheMatrix2.setX(i, i, i);
+ }
+
+ Matrix res = cacheMatrix1.times(cacheMatrix2);
+
+ for(int i = 0; i < MATRIX_SIZE; i++)
+ for(int j = 0; j < MATRIX_SIZE; j++)
+ if (i == j)
+ assertEquals(UNEXPECTED_VAL, i * i, res.get(i, j), PRECISION);
+ else
+ assertEquals(UNEXPECTED_VAL, 0, res.get(i, j), PRECISION);
+ }
+
+ /** */
private void initMtx(Matrix m) {
for (int i = 0; i < m.rowSize(); i++)
for (int j = 0; j < m.columnSize(); j++)