You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ak...@apache.org on 2018/04/11 07:19:55 UTC
[01/14] ignite git commit: IGNITE-8059: Integrate decision tree with
partition based dataset.
Repository: ignite
Updated Branches:
refs/heads/ignite-8201 a80893232 -> df938d562
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
deleted file mode 100644
index f9117f4..0000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
+++ /dev/null
@@ -1,426 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.trees;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledVectorDouble;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.MatrixColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.util.Utils;
-
-/** */
-class SplitDataGenerator<V extends Vector> {
- /** */
- private static final Random rnd = new Random(12349L);
-
- /** */
- private static final double DELTA = 100.0;
-
- /** Map of the form of (is categorical -> list of region indexes). */
- private final Map<Boolean, List<Integer>> di;
-
- /** List of regions. */
- private final List<Region> regs;
-
- /** Data of bounds of regions. */
- private final Map<Integer, IgniteBiTuple<Double, Double>> boundsData;
-
- /** */
- private final Map<Integer, Integer> catFeaturesInfo;
-
- /** Supplier of vectors. */
- private final Supplier<V> supplier;
-
- /** Features count. */
- private final int featCnt;
-
- /**
- * Create SplitDataGenerator.
- *
- * @param featCnt Features count.
- * @param catFeaturesInfo Information about categorical features in form of map (feature index -> categories
- * count).
- * @param supplier Supplier of vectors.
- */
- SplitDataGenerator(int featCnt, Map<Integer, Integer> catFeaturesInfo, Supplier<V> supplier) {
- regs = new LinkedList<>();
- boundsData = new HashMap<>();
- this.supplier = supplier;
- this.featCnt = featCnt;
- this.catFeaturesInfo = catFeaturesInfo;
-
- // Divide indexes into indexes of categorical coordinates and indexes of continuous coordinates.
- di = IntStream.range(0, featCnt).
- boxed().
- collect(Collectors.partitioningBy(catFeaturesInfo::containsKey));
-
- // Categorical coordinates info.
- Map<Integer, CatCoordInfo> catCoords = new HashMap<>();
- di.get(true).forEach(i -> {
- BitSet bs = new BitSet();
- bs.set(0, catFeaturesInfo.get(i));
- catCoords.put(i, new CatCoordInfo(bs));
- });
-
- // Continuous coordinates info.
- Map<Integer, ContCoordInfo> contCoords = new HashMap<>();
- di.get(false).forEach(i -> {
- contCoords.put(i, new ContCoordInfo());
- boundsData.put(i, new IgniteBiTuple<>(-1.0, 1.0));
- });
-
- Region firstReg = new Region(catCoords, contCoords, 0);
- regs.add(firstReg);
- }
-
- /** */
- <D extends ContinuousRegionInfo> void testByGen(int totalPts,
- IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> calc,
- IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> catImpCalc,
- IgniteFunction<DoubleStream, Double> regCalc, Ignite ignite) {
-
- List<IgniteBiTuple<Integer, V>> lst = points(totalPts, (i, rn) -> i).collect(Collectors.toList());
-
- Collections.shuffle(lst, rnd);
-
- SparseDistributedMatrix m = new SparseDistributedMatrix(totalPts,
- featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
- Map<Integer, List<LabeledVectorDouble>> byRegion = new HashMap<>();
-
- int i = 0;
- for (IgniteBiTuple<Integer, V> bt : lst) {
- byRegion.putIfAbsent(bt.get1(), new LinkedList<>());
- byRegion.get(bt.get1()).add(asLabeledVector(bt.get2().getStorage().data()));
- m.setRow(i, bt.get2().getStorage().data());
- i++;
- }
-
- ColumnDecisionTreeTrainer<D> trainer =
- new ColumnDecisionTreeTrainer<>(3, calc, catImpCalc, regCalc, ignite);
-
- DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, catFeaturesInfo));
-
- byRegion.keySet().forEach(k -> mdl.apply(byRegion.get(k).get(0).features()));
- }
-
- /**
- * Split region by continuous coordinate using given threshold.
- *
- * @param regIdx Region index.
- * @param coordIdx Coordinate index.
- * @param threshold Threshold.
- * @return {@code this}.
- */
- SplitDataGenerator<V> split(int regIdx, int coordIdx, double threshold) {
- Region regToSplit = regs.get(regIdx);
- ContCoordInfo cci = regToSplit.contCoords.get(coordIdx);
-
- double left = cci.left;
- double right = cci.right;
-
- if (threshold < left || threshold > right)
- throw new MathIllegalArgumentException("Threshold is out of region bounds.");
-
- regToSplit.incTwoPow();
-
- Region newReg = Utils.copy(regToSplit);
- newReg.contCoords.get(coordIdx).left = threshold;
-
- regs.add(regIdx + 1, newReg);
- cci.right = threshold;
-
- IgniteBiTuple<Double, Double> bounds = boundsData.get(coordIdx);
- double min = bounds.get1();
- double max = bounds.get2();
- boundsData.put(coordIdx, new IgniteBiTuple<>(Math.min(threshold, min), Math.max(max, threshold)));
-
- return this;
- }
-
- /**
- * Split region by categorical coordinate.
- *
- * @param regIdx Region index.
- * @param coordIdx Coordinate index.
- * @param cats Categories allowed for the left sub region.
- * @return {@code this}.
- */
- SplitDataGenerator<V> split(int regIdx, int coordIdx, int[] cats) {
- BitSet subset = new BitSet();
- Arrays.stream(cats).forEach(subset::set);
- Region regToSplit = regs.get(regIdx);
- CatCoordInfo cci = regToSplit.catCoords.get(coordIdx);
-
- BitSet ssc = (BitSet)subset.clone();
- BitSet set = cci.bs;
- ssc.and(set);
- if (ssc.length() != subset.length())
- throw new MathIllegalArgumentException("Splitter set is not a subset of a parent subset.");
-
- ssc.xor(set);
- set.and(subset);
-
- regToSplit.incTwoPow();
- Region newReg = Utils.copy(regToSplit);
- newReg.catCoords.put(coordIdx, new CatCoordInfo(ssc));
-
- regs.add(regIdx + 1, newReg);
-
- return this;
- }
-
- /**
- * Get stream of points generated by this generator.
- *
- * @param ptsCnt Points count.
- */
- private Stream<IgniteBiTuple<Integer, V>> points(int ptsCnt, BiFunction<Double, Random, Double> f) {
- return IntStream.range(0, regs.size()).
- boxed().
- map(i -> regs.get(i).generatePoints(ptsCnt, f.apply((double)i, rnd), boundsData, di, supplier, rnd)
- .map(v -> new IgniteBiTuple<>(i, v))).flatMap(Function.identity());
- }
-
- /**
- * Convert double array to {@link LabeledVectorDouble}
- *
- * @param arr Array for conversion.
- * @return LabeledVectorDouble.
- */
- private static LabeledVectorDouble<DenseLocalOnHeapVector> asLabeledVector(double arr[]) {
- return new LabeledVectorDouble<>(new DenseLocalOnHeapVector(
- Arrays.copyOf(arr, arr.length - 1)), arr[arr.length - 1]);
- }
-
- /**
- * Categorical coordinate info.
- */
- private static class CatCoordInfo implements Serializable {
- /**
- * Defines categories which are included in this region
- */
- private final BitSet bs;
-
- /**
- * Construct CatCoordInfo.
- *
- * @param bs Bitset.
- */
- CatCoordInfo(BitSet bs) {
- this.bs = bs;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "CatCoordInfo [" +
- "bs=" + bs +
- ']';
- }
- }
-
- /**
- * Continuous coordinate info.
- */
- private static class ContCoordInfo implements Serializable {
- /**
- * Left (min) bound of region.
- */
- private double left;
-
- /**
- * Right (max) bound of region.
- */
- private double right;
-
- /**
- * Construct ContCoordInfo.
- */
- ContCoordInfo() {
- left = Double.NEGATIVE_INFINITY;
- right = Double.POSITIVE_INFINITY;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "ContCoordInfo [" +
- "left=" + left +
- ", right=" + right +
- ']';
- }
- }
-
- /**
- * Class representing information about region.
- */
- private static class Region implements Serializable {
- /**
- * Information about categorical coordinates restrictions of this region in form of
- * (coordinate index -> restriction)
- */
- private final Map<Integer, CatCoordInfo> catCoords;
-
- /**
- * Information about continuous coordinates restrictions of this region in form of
- * (coordinate index -> restriction)
- */
- private final Map<Integer, ContCoordInfo> contCoords;
-
- /**
- * Region should contain {@code 1/2^twoPow * totalPoints} points.
- */
- private int twoPow;
-
- /**
- * Construct region by information about restrictions on coordinates (features) values.
- *
- * @param catCoords Restrictions on categorical coordinates.
- * @param contCoords Restrictions on continuous coordinates
- * @param twoPow Region should contain {@code 1/2^twoPow * totalPoints} points.
- */
- Region(Map<Integer, CatCoordInfo> catCoords, Map<Integer, ContCoordInfo> contCoords, int twoPow) {
- this.catCoords = catCoords;
- this.contCoords = contCoords;
- this.twoPow = twoPow;
- }
-
- /** */
- int divideBy() {
- return 1 << twoPow;
- }
-
- /** */
- void incTwoPow() {
- twoPow++;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "Region [" +
- "catCoords=" + catCoords +
- ", contCoords=" + contCoords +
- ", twoPow=" + twoPow +
- ']';
- }
-
- /**
- * Generate continuous coordinate for this region.
- *
- * @param coordIdx Coordinate index.
- * @param boundsData Data with bounds
- * @param rnd Random numbers generator.
- * @return Categorical coordinate value.
- */
- double generateContCoord(int coordIdx, Map<Integer, IgniteBiTuple<Double, Double>> boundsData,
- Random rnd) {
- ContCoordInfo cci = contCoords.get(coordIdx);
- double left = cci.left;
- double right = cci.right;
-
- if (left == Double.NEGATIVE_INFINITY)
- left = boundsData.get(coordIdx).get1() - DELTA;
-
- if (right == Double.POSITIVE_INFINITY)
- right = boundsData.get(coordIdx).get2() + DELTA;
-
- double size = right - left;
-
- return left + rnd.nextDouble() * size;
- }
-
- /**
- * Generate categorical coordinate value for this region.
- *
- * @param coordIdx Coordinate index.
- * @param rnd Random numbers generator.
- * @return Categorical coordinate value.
- */
- double generateCatCoord(int coordIdx, Random rnd) {
- // Pick random bit.
- BitSet bs = catCoords.get(coordIdx).bs;
- int j = rnd.nextInt(bs.length());
-
- int i = 0;
- int bn = 0;
- int bnp = 0;
-
- while ((bn = bs.nextSetBit(bn)) != -1 && i <= j) {
- i++;
- bnp = bn;
- bn++;
- }
-
- return bnp;
- }
-
- /**
- * Generate points for this region.
- *
- * @param ptsCnt Count of points to generate.
- * @param val Label for all points in this region.
- * @param boundsData Data about bounds of continuous coordinates.
- * @param catCont Data about which categories can be in this region in the form (coordinate index -> list of
- * categories indexes).
- * @param s Vectors supplier.
- * @param rnd Random numbers generator.
- * @param <V> Type of vectors.
- * @return Stream of generated points for this region.
- */
- <V extends Vector> Stream<V> generatePoints(int ptsCnt, double val,
- Map<Integer, IgniteBiTuple<Double, Double>> boundsData, Map<Boolean, List<Integer>> catCont,
- Supplier<V> s,
- Random rnd) {
- return IntStream.range(0, ptsCnt / divideBy()).mapToObj(i -> {
- V v = s.get();
- int coordsCnt = v.size();
- catCont.get(false).forEach(ci -> v.setX(ci, generateContCoord(ci, boundsData, rnd)));
- catCont.get(true).forEach(ci -> v.setX(ci, generateCatCoord(ci, rnd)));
-
- v.setX(coordsCnt - 1, val);
- return v;
- });
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/package-info.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/package-info.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/package-info.java
deleted file mode 100644
index fc379a6..0000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * ML Grid decision tree benchmarks.
- */
-package org.apache.ignite.yardstick.ml.trees;
\ No newline at end of file
[11/14] ignite git commit: IGNITE-6430 Complete failing test early
Posted by ak...@apache.org.
IGNITE-6430 Complete failing test early
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3a717658
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3a717658
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3a717658
Branch: refs/heads/ignite-8201
Commit: 3a7176582d622fc8a52a730f2a3f1e4b4319e4a3
Parents: 9bb4ce8
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Tue Apr 10 20:33:47 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Apr 10 20:33:47 2018 +0300
----------------------------------------------------------------------
.../cache/CacheGroupsMetricsRebalanceTest.java | 31 ++++++++++++--------
1 file changed, 18 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3a717658/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java
index 89c8236..ceb9852 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java
@@ -227,29 +227,34 @@ public class CacheGroupsMetricsRebalanceTest extends GridCommonAbstractTest {
System.out.println("Wait until keys left will be less " + keysLine);
- while (finishRebalanceLatch.getCount() != 0) {
- CacheMetrics m = ig2.cache(CACHE1).localMetrics();
+ try {
+ while (finishRebalanceLatch.getCount() != 0) {
+ CacheMetrics m = ig2.cache(CACHE1).localMetrics();
- long keyLeft = m.getKeysToRebalanceLeft();
+ long keyLeft = m.getKeysToRebalanceLeft();
- if (keyLeft > 0 && keyLeft < keysLine)
- latch.countDown();
+ if (keyLeft > 0 && keyLeft < keysLine)
+ latch.countDown();
- System.out.println("Keys left: " + m.getKeysToRebalanceLeft());
+ System.out.println("Keys left: " + m.getKeysToRebalanceLeft());
- try {
- Thread.sleep(1_000);
- }
- catch (InterruptedException e) {
- System.out.println("Interrupt thread: " + e.getMessage());
+ try {
+ Thread.sleep(1_000);
+ }
+ catch (InterruptedException e) {
+ System.out.println("Interrupt thread: " + e.getMessage());
- Thread.currentThread().interrupt();
+ Thread.currentThread().interrupt();
+ }
}
}
+ finally {
+ latch.countDown();
+ }
}
});
- latch.await();
+ assertTrue(latch.await(getTestTimeout(), TimeUnit.MILLISECONDS));
long finishTime = ig2.cache(CACHE1).localMetrics().getEstimatedRebalancingFinishTime();
[05/14] ignite git commit: IGNITE-8059: Integrate decision tree with
partition based dataset.
Posted by ak...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasure.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasure.java
new file mode 100644
index 0000000..3fc8515
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasure.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.mse;
+
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Mean squared error (variance) impurity measure which is calculated the following way:
+ * {@code \frac{1}{L}\sum_{i=0}^{n}(y_i - \mu)^2}.
+ */
+public class MSEImpurityMeasure implements ImpurityMeasure<MSEImpurityMeasure> {
+ /** */
+ private static final long serialVersionUID = 4536394578628409689L;
+
+ /** Sum of all elements in the left part. */
+ private final double leftY;
+
+ /** Sum of all squared elements in the left part. */
+ private final double leftY2;
+
+ /** Number of elements in the left part. */
+ private final long leftCnt;
+
+ /** Sum of all elements in the right part. */
+ private final double rightY;
+
+ /** Sum of all squared elements in the right part. */
+ private final double rightY2;
+
+ /** Number of elements in the right part. */
+ private final long rightCnt;
+
+ /**
+ * Constructs a new instance of mean squared error (variance) impurity measure.
+ *
+ * @param leftY Sum of all elements in the left part.
+ * @param leftY2 Sum of all squared elements in the left part.
+ * @param leftCnt Number of elements in the left part.
+ * @param rightY Sum of all elements in the right part.
+ * @param rightY2 Sum of all squared elements in the right part.
+ * @param rightCnt Number of elements in the right part.
+ */
+ public MSEImpurityMeasure(double leftY, double leftY2, long leftCnt, double rightY, double rightY2, long rightCnt) {
+ this.leftY = leftY;
+ this.leftY2 = leftY2;
+ this.leftCnt = leftCnt;
+ this.rightY = rightY;
+ this.rightY2 = rightY2;
+ this.rightCnt = rightCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public double impurity() {
+ double impurity = 0;
+
+ if (leftCnt > 0)
+ impurity += leftY2 - 2.0 * leftY / leftCnt * leftY + Math.pow(leftY / leftCnt, 2) * leftCnt;
+
+ if (rightCnt > 0)
+ impurity += rightY2 - 2.0 * rightY / rightCnt * rightY + Math.pow(rightY / rightCnt, 2) * rightCnt;
+
+ return impurity;
+ }
+
+ /** {@inheritDoc} */
+ @Override public MSEImpurityMeasure add(MSEImpurityMeasure b) {
+ return new MSEImpurityMeasure(
+ leftY + b.leftY,
+ leftY2 + b.leftY2,
+ leftCnt + b.leftCnt,
+ rightY + b.rightY,
+ rightY2 + b.rightY2,
+ rightCnt + b.rightCnt
+ );
+ }
+
+ /** {@inheritDoc} */
+ @Override public MSEImpurityMeasure subtract(MSEImpurityMeasure b) {
+ return new MSEImpurityMeasure(
+ leftY - b.leftY,
+ leftY2 - b.leftY2,
+ leftCnt - b.leftCnt,
+ rightY - b.rightY,
+ rightY2 - b.rightY2,
+ rightCnt - b.rightCnt
+ );
+ }
+
+ /** */
+ public double getLeftY() {
+ return leftY;
+ }
+
+ /** */
+ public double getLeftY2() {
+ return leftY2;
+ }
+
+ /** */
+ public long getLeftCnt() {
+ return leftCnt;
+ }
+
+ /** */
+ public double getRightY() {
+ return rightY;
+ }
+
+ /** */
+ public double getRightY2() {
+ return rightY2;
+ }
+
+ /** */
+ public long getRightCnt() {
+ return rightCnt;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculator.java
new file mode 100644
index 0000000..cb5019c
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculator.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.mse;
+
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+
+/**
+ * Meas squared error (variance) impurity measure calculator.
+ */
+public class MSEImpurityMeasureCalculator implements ImpurityMeasureCalculator<MSEImpurityMeasure> {
+ /** */
+ private static final long serialVersionUID = 288747414953756824L;
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public StepFunction<MSEImpurityMeasure>[] calculate(DecisionTreeData data) {
+ double[][] features = data.getFeatures();
+ double[] labels = data.getLabels();
+
+ if (features.length > 0) {
+ StepFunction<MSEImpurityMeasure>[] res = new StepFunction[features[0].length];
+
+ for (int col = 0; col < res.length; col++) {
+ data.sort(col);
+
+ double[] x = new double[features.length + 1];
+ MSEImpurityMeasure[] y = new MSEImpurityMeasure[features.length + 1];
+
+ x[0] = Double.NEGATIVE_INFINITY;
+
+ for (int leftSize = 0; leftSize <= features.length; leftSize++) {
+ double leftY = 0;
+ double leftY2 = 0;
+ double rightY = 0;
+ double rightY2 = 0;
+
+ for (int i = 0; i < leftSize; i++) {
+ leftY += labels[i];
+ leftY2 += Math.pow(labels[i], 2);
+ }
+
+ for (int i = leftSize; i < features.length; i++) {
+ rightY += labels[i];
+ rightY2 += Math.pow(labels[i], 2);
+ }
+
+ if (leftSize < features.length)
+ x[leftSize + 1] = features[leftSize][col];
+
+ y[leftSize] = new MSEImpurityMeasure(
+ leftY, leftY2, leftSize, rightY, rightY2, features.length - leftSize
+ );
+ }
+
+ res[col] = new StepFunction<>(x, y);
+ }
+
+ return res;
+ }
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/package-info.java
new file mode 100644
index 0000000..23ec4e0
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains mean squared error impurity measure and calculator.
+ */
+package org.apache.ignite.ml.tree.impurity.mse;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/package-info.java
new file mode 100644
index 0000000..4155593
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Root package for decision tree impurity measures and calculators.
+ */
+package org.apache.ignite.ml.tree.impurity;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressor.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressor.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressor.java
new file mode 100644
index 0000000..2418571
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressor.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Simple step function compressor.
+ *
+ * @param <T> Type of step function values.
+ */
+public class SimpleStepFunctionCompressor<T extends ImpurityMeasure<T>> implements StepFunctionCompressor<T> {
+ /** */
+ private static final long serialVersionUID = -3231787633598409157L;
+
+ /** Min size of step function to be compressed. */
+ private final int minSizeToBeCompressed;
+
+ /** In case of compression min impurity increase that will be recorded. */
+ private final double minImpurityIncreaseForRecord;
+
+ /** In case of compression min impurity decrease that will be recorded. */
+ private final double minImpurityDecreaseForRecord;
+
+ /**
+ * Constructs a new instance of simple step function compressor with default parameters.
+ */
+ public SimpleStepFunctionCompressor() {
+ this(10, 0.1, 0.05);
+ }
+
+ /**
+ * Constructs a new instance of simple step function compressor.
+ *
+ * @param minSizeToBeCompressed Min size of step function to be compressed.
+ * @param minImpurityIncreaseForRecord In case of compression min impurity increase that will be recorded.
+ * @param minImpurityDecreaseForRecord In case of compression min impurity decrease that will be recorded.
+ */
+ public SimpleStepFunctionCompressor(int minSizeToBeCompressed, double minImpurityIncreaseForRecord,
+ double minImpurityDecreaseForRecord) {
+ this.minSizeToBeCompressed = minSizeToBeCompressed;
+ this.minImpurityIncreaseForRecord = minImpurityIncreaseForRecord;
+ this.minImpurityDecreaseForRecord = minImpurityDecreaseForRecord;
+ }
+
+ /** {@inheritDoc} */
+ @Override public StepFunction<T> compress(StepFunction<T> function) {
+ double[] arguments = function.getX();
+ T[] values = function.getY();
+
+ if (arguments.length >= minSizeToBeCompressed) {
+ List<StepFunctionPoint> points = new ArrayList<>();
+
+ for (int i = 0; i < arguments.length; i++)
+ points.add(new StepFunctionPoint(arguments[i], values[i]));
+
+ points = compress(points);
+
+ double[] resX = new double[points.size()];
+ T[] resY = Arrays.copyOf(values, points.size());
+
+ for (int i = 0; i < points.size(); i++) {
+ StepFunctionPoint pnt = points.get(i);
+ resX[i] = pnt.x;
+ resY[i] = pnt.y;
+ }
+
+ return new StepFunction<>(resX, resY);
+ }
+
+ return function;
+ }
+
+ /**
+ * Compresses list of step function points.
+ *
+ * @param points Step function points.
+ * @return Compressed step function points.
+ */
+ private List<StepFunctionPoint> compress(List<StepFunctionPoint> points) {
+ List<StepFunctionPoint> res = new ArrayList<>();
+
+ double minImpurity = Double.MAX_VALUE, maxImpurity = Double.MIN_VALUE;
+ for (int i = 0; i < points.size(); i++) {
+ StepFunctionPoint pnt = points.get(i);
+
+ double impurity = pnt.y.impurity();
+
+ if (impurity > maxImpurity)
+ maxImpurity = impurity;
+
+ if (impurity < minImpurity)
+ minImpurity = impurity;
+ }
+
+ Double prev = null;
+ for (StepFunctionPoint pnt : points) {
+ double impurity = (pnt.y.impurity() - minImpurity) / (maxImpurity - minImpurity);
+ if (prev == null ||
+ prev - impurity >= minImpurityDecreaseForRecord ||
+ impurity - prev >= minImpurityIncreaseForRecord) {
+ prev = impurity;
+ res.add(pnt);
+ }
+ }
+
+ return res;
+ }
+
+ /**
+ * Util class that represents step function point.
+ */
+ private class StepFunctionPoint {
+ /** Argument of the step start. */
+ private final double x;
+
+ /** Value of the step. */
+ private final T y;
+
+ /**
+ * Constructs a new instance of util class that represents step function point.
+ *
+ * @param x Argument of the step start.
+ * @param y Value of the step.
+ */
+ StepFunctionPoint(double x, T y) {
+ this.x = x;
+ this.y = y;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunction.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunction.java
new file mode 100644
index 0000000..431503d
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunction.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import java.util.Arrays;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Step function described by {@code x} and {@code y} points.
+ *
+ * @param <T> Type of function values.
+ */
+public class StepFunction<T extends ImpurityMeasure<T>> {
+ /** Argument of every steps start. Should be ascendingly sorted all the time. */
+ private final double[] x;
+
+ /** Value of every step. */
+ private final T[] y;
+
+ /**
+ * Constructs a new instance of step function.
+ *
+ * @param x Argument of every steps start.
+ * @param y Value of every step.
+ */
+ public StepFunction(double[] x, T[] y) {
+ assert x.length == y.length : "Argument and value arrays have to be the same length";
+
+ this.x = x;
+ this.y = y;
+
+ sort(x, y, 0, x.length - 1);
+ }
+
+ /**
+ * Adds the given step function to this.
+ *
+ * @param b Another step function.
+ * @return Sum of this and the given function.
+ */
+ public StepFunction<T> add(StepFunction<T> b) {
+ int resSize = 0, leftPtr = 0, rightPtr = 0;
+ double previousPnt = 0;
+
+ while (leftPtr < x.length || rightPtr < b.x.length) {
+ if (rightPtr >= b.x.length || (leftPtr < x.length && x[leftPtr] < b.x[rightPtr])) {
+ if (resSize == 0 || x[leftPtr] != previousPnt) {
+ previousPnt = x[leftPtr];
+ resSize++;
+ }
+
+ leftPtr++;
+ }
+ else {
+ if (resSize == 0 || b.x[rightPtr] != previousPnt) {
+ previousPnt = b.x[rightPtr];
+ resSize++;
+ }
+
+ rightPtr++;
+ }
+ }
+
+ double[] resX = new double[resSize];
+ T[] resY = Arrays.copyOf(y, resSize);
+
+ leftPtr = 0;
+ rightPtr = 0;
+
+ for (int i = 0; leftPtr < x.length || rightPtr < b.x.length; i++) {
+ if (rightPtr >= b.x.length || (leftPtr < x.length && x[leftPtr] < b.x[rightPtr])) {
+ boolean override = i > 0 && x[leftPtr] == resX[i - 1];
+ int target = override ? i - 1 : i;
+
+ resY[target] = override ? resY[target] : null;
+ resY[target] = i > 0 ? resY[i - 1] : null;
+ resY[target] = resY[target] == null ? y[leftPtr] : resY[target].add(y[leftPtr]);
+
+ if (leftPtr > 0)
+ resY[target] = resY[target].subtract(y[leftPtr - 1]);
+
+ resX[target] = x[leftPtr];
+ i = target;
+
+ leftPtr++;
+ }
+ else {
+ boolean override = i > 0 && b.x[rightPtr] == resX[i - 1];
+ int target = override ? i - 1 : i;
+
+ resY[target] = override ? resY[target] : null;
+ resY[target] = i > 0 ? resY[i - 1] : null;
+
+ resY[target] = resY[target] == null ? b.y[rightPtr] : resY[target].add(b.y[rightPtr]);
+
+ if (rightPtr > 0)
+ resY[target] = resY[target].subtract(b.y[rightPtr - 1]);
+
+ resX[target] = b.x[rightPtr];
+ i = target;
+
+ rightPtr++;
+ }
+ }
+
+ return new StepFunction<>(resX, resY);
+ }
+
+ /** */
+ private void sort(double[] x, T[] y, int from, int to) {
+ if (from < to) {
+ double pivot = x[(from + to) / 2];
+
+ int i = from, j = to;
+ while (i <= j) {
+ while (x[i] < pivot) i++;
+ while (x[j] > pivot) j--;
+
+ if (i <= j) {
+ double tmpX = x[i];
+ x[i] = x[j];
+ x[j] = tmpX;
+
+ T tmpY = y[i];
+ y[i] = y[j];
+ y[j] = tmpY;
+
+ i++;
+ j--;
+ }
+ }
+
+ sort(x, y, from, j);
+ sort(x, y, i, to);
+ }
+ }
+
+ /** */
+ public double[] getX() {
+ return x;
+ }
+
+ /** */
+ public T[] getY() {
+ return y;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionCompressor.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionCompressor.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionCompressor.java
new file mode 100644
index 0000000..41baa29
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionCompressor.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Base interface for step function compressors which reduces step function size.
+ *
+ * @param <T> Type of step function value.
+ */
+public interface StepFunctionCompressor<T extends ImpurityMeasure<T>> extends Serializable {
+ /**
+ * Compresses the given step function.
+ *
+ * @param function Step function.
+ * @return Compressed step function.
+ */
+ public StepFunction<T> compress(StepFunction<T> function);
+
+ /**
+ * Compresses every step function in the given array.
+ *
+ * @param functions Array of step functions.
+ * @return Arrays of compressed step function.
+ */
+ default public StepFunction<T>[] compress(StepFunction<T>[] functions) {
+ if (functions == null)
+ return null;
+
+ StepFunction<T>[] res = Arrays.copyOf(functions, functions.length);
+
+ for (int i = 0; i < res.length; i++)
+ res[i] = compress(res[i]);
+
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/package-info.java
new file mode 100644
index 0000000..99df618
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains util classes used in decision tree impurity calculators.
+ */
+package org.apache.ignite.ml.tree.impurity.util;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/DecisionTreeLeafBuilder.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/DecisionTreeLeafBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/DecisionTreeLeafBuilder.java
new file mode 100644
index 0000000..976e30d
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/DecisionTreeLeafBuilder.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.leaf;
+
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.DecisionTreeLeafNode;
+import org.apache.ignite.ml.tree.TreeFilter;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+
+/**
+ * Base interface for decision tree leaf builders.
+ */
+public interface DecisionTreeLeafBuilder {
+ /**
+ * Creates new leaf node for given dataset and node predicate.
+ *
+ * @param dataset Dataset.
+ * @param pred Node predicate.
+ * @return Leaf node.
+ */
+ public DecisionTreeLeafNode createLeafNode(Dataset<EmptyContext, DecisionTreeData> dataset, TreeFilter pred);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MeanDecisionTreeLeafBuilder.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MeanDecisionTreeLeafBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MeanDecisionTreeLeafBuilder.java
new file mode 100644
index 0000000..2e05215
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MeanDecisionTreeLeafBuilder.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.leaf;
+
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.DecisionTreeLeafNode;
+import org.apache.ignite.ml.tree.TreeFilter;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+
+/**
+ * Decision tree leaf node builder that chooses mean value as a leaf value.
+ */
+public class MeanDecisionTreeLeafBuilder implements DecisionTreeLeafBuilder {
+ /** {@inheritDoc} */
+ @Override public DecisionTreeLeafNode createLeafNode(Dataset<EmptyContext, DecisionTreeData> dataset,
+ TreeFilter pred) {
+ double[] aa = dataset.compute(part -> {
+ double mean = 0;
+ int cnt = 0;
+
+ for (int i = 0; i < part.getFeatures().length; i++) {
+ if (pred.test(part.getFeatures()[i])) {
+ mean += part.getLabels()[i];
+ cnt++;
+ }
+ }
+
+ if (cnt != 0) {
+ mean = mean / cnt;
+
+ return new double[] {mean, cnt};
+ }
+
+ return null;
+ }, this::reduce);
+
+ return aa != null ? new DecisionTreeLeafNode(aa[0]) : null;
+ }
+
+ /** */
+ private double[] reduce(double[] a, double[] b) {
+ if (a == null)
+ return b;
+ else if (b == null)
+ return a;
+ else {
+ double aMean = a[0];
+ double aCnt = a[1];
+ double bMean = b[0];
+ double bCnt = b[1];
+
+ double mean = (aMean * aCnt + bMean * bCnt) / (aCnt + bCnt);
+
+ return new double[] {mean, aCnt + bCnt};
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MostCommonDecisionTreeLeafBuilder.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MostCommonDecisionTreeLeafBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MostCommonDecisionTreeLeafBuilder.java
new file mode 100644
index 0000000..1e8b941
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MostCommonDecisionTreeLeafBuilder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.leaf;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.DecisionTreeLeafNode;
+import org.apache.ignite.ml.tree.TreeFilter;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+
+/**
+ * Decision tree leaf node builder that chooses most common value as a leaf node value.
+ */
+public class MostCommonDecisionTreeLeafBuilder implements DecisionTreeLeafBuilder {
+ /** {@inheritDoc} */
+ @Override public DecisionTreeLeafNode createLeafNode(Dataset<EmptyContext, DecisionTreeData> dataset,
+ TreeFilter pred) {
+ Map<Double, Integer> cnt = dataset.compute(part -> {
+
+ if (part.getFeatures() != null) {
+ Map<Double, Integer> map = new HashMap<>();
+
+ for (int i = 0; i < part.getFeatures().length; i++) {
+ if (pred.test(part.getFeatures()[i])) {
+ double lb = part.getLabels()[i];
+
+ if (map.containsKey(lb))
+ map.put(lb, map.get(lb) + 1);
+ else
+ map.put(lb, 1);
+ }
+ }
+
+ return map;
+ }
+
+ return null;
+ }, this::reduce);
+
+ double bestVal = 0;
+ int bestCnt = -1;
+
+ for (Map.Entry<Double, Integer> e : cnt.entrySet()) {
+ if (e.getValue() > bestCnt) {
+ bestCnt = e.getValue();
+ bestVal = e.getKey();
+ }
+ }
+
+ return new DecisionTreeLeafNode(bestVal);
+ }
+
+ /** */
+ private Map<Double, Integer> reduce(Map<Double, Integer> a, Map<Double, Integer> b) {
+ if (a == null)
+ return b;
+ else if (b == null)
+ return a;
+ else {
+ for (Map.Entry<Double, Integer> e : b.entrySet()) {
+ if (a.containsKey(e.getKey()))
+ a.put(e.getKey(), a.get(e.getKey()) + e.getValue());
+ else
+ a.put(e.getKey(), e.getValue());
+ }
+ return a;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/package-info.java
new file mode 100644
index 0000000..26ec67d
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Root package for decision trees leaf builders.
+ */
+package org.apache.ignite.ml.tree.leaf;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/package-info.java
new file mode 100644
index 0000000..660f3f3
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Root package for decision trees.
+ */
+package org.apache.ignite.ml.tree;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalRegionInfo.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalRegionInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalRegionInfo.java
deleted file mode 100644
index 3ae474e..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalRegionInfo.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.BitSet;
-
-/**
- * Information about categorical region.
- */
-public class CategoricalRegionInfo extends RegionInfo implements Externalizable {
- /**
- * Bitset representing categories of this region.
- */
- private BitSet cats;
-
- /**
- * @param impurity Impurity of region.
- * @param cats Bitset representing categories of this region.
- */
- public CategoricalRegionInfo(double impurity, BitSet cats) {
- super(impurity);
-
- this.cats = cats;
- }
-
- /**
- * No-op constructor for serialization/deserialization.
- */
- public CategoricalRegionInfo() {
- // No-op
- }
-
- /**
- * Get bitset representing categories of this region.
- *
- * @return Bitset representing categories of this region.
- */
- public BitSet cats() {
- return cats;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- super.writeExternal(out);
- out.writeObject(cats);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- super.readExternal(in);
- cats = (BitSet)in.readObject();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalSplitInfo.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalSplitInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalSplitInfo.java
deleted file mode 100644
index 94cb1e8..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalSplitInfo.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.BitSet;
-import org.apache.ignite.ml.trees.nodes.CategoricalSplitNode;
-import org.apache.ignite.ml.trees.nodes.SplitNode;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-
-/**
- * Information about split of categorical feature.
- *
- * @param <D> Class representing information of left and right subregions.
- */
-public class CategoricalSplitInfo<D extends RegionInfo> extends SplitInfo<D> {
- /** Bitset indicating which vectors are assigned to left subregion. */
- private final BitSet bs;
-
- /**
- * @param regionIdx Index of region which is split.
- * @param leftData Data of left subregion.
- * @param rightData Data of right subregion.
- * @param bs Bitset indicating which vectors are assigned to left subregion.
- */
- public CategoricalSplitInfo(int regionIdx, D leftData, D rightData,
- BitSet bs) {
- super(regionIdx, leftData, rightData);
- this.bs = bs;
- }
-
- /** {@inheritDoc} */
- @Override public SplitNode createSplitNode(int featureIdx) {
- return new CategoricalSplitNode(featureIdx, bs);
- }
-
- /**
- * Get bitset indicating which vectors are assigned to left subregion.
- */
- public BitSet bitSet() {
- return bs;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "CategoricalSplitInfo [" +
- "infoGain=" + infoGain +
- ", regionIdx=" + regionIdx +
- ", leftData=" + leftData +
- ", bs=" + bs +
- ", rightData=" + rightData +
- ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousRegionInfo.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousRegionInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousRegionInfo.java
deleted file mode 100644
index e98bb72..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousRegionInfo.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-
-/**
- * Information about region used by continuous features.
- */
-public class ContinuousRegionInfo extends RegionInfo {
- /**
- * Count of samples in this region.
- */
- private int size;
-
- /**
- * @param impurity Impurity of the region.
- * @param size Size of this region
- */
- public ContinuousRegionInfo(double impurity, int size) {
- super(impurity);
- this.size = size;
- }
-
- /**
- * No-op constructor for serialization/deserialization.
- */
- public ContinuousRegionInfo() {
- // No-op
- }
-
- /**
- * Get the size of region.
- */
- public int getSize() {
- return size;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "ContinuousRegionInfo [" +
- "size=" + size +
- ']';
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- super.writeExternal(out);
- out.writeInt(size);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- super.readExternal(in);
- size = in.readInt();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousSplitCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousSplitCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousSplitCalculator.java
deleted file mode 100644
index 3a0e9da..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousSplitCalculator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.ContinuousFeatureProcessor;
-
-/**
- * This class is used for calculation of best split by continuous feature.
- *
- * @param <C> Class in which information about region will be stored.
- */
-public interface ContinuousSplitCalculator<C extends ContinuousRegionInfo> {
- /**
- * Calculate region info 'from scratch'.
- *
- * @param s Stream of labels in this region.
- * @param l Index of sample projection on this feature in array sorted by this projection value and intervals
- * bitsets. ({@link ContinuousFeatureProcessor}).
- * @return Region info.
- */
- C calculateRegionInfo(DoubleStream s, int l);
-
- /**
- * Calculate split info of best split of region given information about this region.
- *
- * @param sampleIndexes Indexes of samples of this region.
- * @param values All values of this feature.
- * @param labels All labels of this feature.
- * @param regionIdx Index of region being split.
- * @param data Information about region being split which can be used for computations.
- * @return Information about best split of region with index given by regionIdx.
- */
- SplitInfo<C> splitRegion(Integer[] sampleIndexes, double[] values, double[] labels, int regionIdx, C data);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/RegionInfo.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/RegionInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/RegionInfo.java
deleted file mode 100644
index 8ec7db3..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/RegionInfo.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-
-/** Class containing information about region. */
-public class RegionInfo implements Externalizable {
- /** Impurity in this region. */
- private double impurity;
-
- /**
- * @param impurity Impurity of this region.
- */
- public RegionInfo(double impurity) {
- this.impurity = impurity;
- }
-
- /**
- * No-op constructor for serialization/deserialization.
- */
- public RegionInfo() {
- // No-op
- }
-
- /**
- * Get impurity in this region.
- *
- * @return Impurity of this region.
- */
- public double impurity() {
- return impurity;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeDouble(impurity);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- impurity = in.readDouble();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/DecisionTreeModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/DecisionTreeModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/DecisionTreeModel.java
deleted file mode 100644
index 572e64a..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/DecisionTreeModel.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.models;
-
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.trees.nodes.DecisionTreeNode;
-
-/**
- * Model for decision tree.
- */
-public class DecisionTreeModel implements Model<Vector, Double> {
- /** Root node of the decision tree. */
- private final DecisionTreeNode root;
-
- /**
- * Construct decision tree model.
- *
- * @param root Root of decision tree.
- */
- public DecisionTreeModel(DecisionTreeNode root) {
- this.root = root;
- }
-
- /** {@inheritDoc} */
- @Override public Double apply(Vector val) {
- return root.process(val);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/package-info.java
deleted file mode 100644
index ce8418e..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains decision tree models.
- */
-package org.apache.ignite.ml.trees.models;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/CategoricalSplitNode.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/CategoricalSplitNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/CategoricalSplitNode.java
deleted file mode 100644
index cae6d4a..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/CategoricalSplitNode.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.nodes;
-
-import java.util.BitSet;
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Split node by categorical feature.
- */
-public class CategoricalSplitNode extends SplitNode {
- /** Bitset specifying which categories belong to left subregion. */
- private final BitSet bs;
-
- /**
- * Construct categorical split node.
- *
- * @param featureIdx Index of feature by which split is done.
- * @param bs Bitset specifying which categories go to the left subtree.
- */
- public CategoricalSplitNode(int featureIdx, BitSet bs) {
- super(featureIdx);
- this.bs = bs;
- }
-
- /** {@inheritDoc} */
- @Override public boolean goLeft(Vector v) {
- return bs.get((int)v.getX(featureIdx));
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "CategoricalSplitNode [bs=" + bs + ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/ContinuousSplitNode.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/ContinuousSplitNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/ContinuousSplitNode.java
deleted file mode 100644
index 285cfcd..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/ContinuousSplitNode.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.nodes;
-
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Split node representing split of continuous feature.
- */
-public class ContinuousSplitNode extends SplitNode {
- /** Threshold. Values which are less or equal then threshold are assigned to the left subregion. */
- private final double threshold;
-
- /**
- * Construct ContinuousSplitNode by threshold and feature index.
- *
- * @param threshold Threshold.
- * @param featureIdx Feature index.
- */
- public ContinuousSplitNode(double threshold, int featureIdx) {
- super(featureIdx);
- this.threshold = threshold;
- }
-
- /** {@inheritDoc} */
- @Override public boolean goLeft(Vector v) {
- return v.getX(featureIdx) <= threshold;
- }
-
- /** Threshold. Values which are less or equal then threshold are assigned to the left subregion. */
- public double threshold() {
- return threshold;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "ContinuousSplitNode [" +
- "threshold=" + threshold +
- ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/DecisionTreeNode.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/DecisionTreeNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/DecisionTreeNode.java
deleted file mode 100644
index d31623d..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/DecisionTreeNode.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.nodes;
-
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Node of decision tree.
- */
-public interface DecisionTreeNode {
- /**
- * Assign the double value to the given vector.
- *
- * @param v Vector.
- * @return Value assigned to the given vector.
- */
- double process(Vector v);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/Leaf.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/Leaf.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/Leaf.java
deleted file mode 100644
index 79b441f..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/Leaf.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.nodes;
-
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Terminal node of the decision tree.
- */
-public class Leaf implements DecisionTreeNode {
- /**
- * Value in subregion represented by this node.
- */
- private final double val;
-
- /**
- * Construct the leaf of decision tree.
- *
- * @param val Value in subregion represented by this node.
- */
- public Leaf(double val) {
- this.val = val;
- }
-
- /**
- * Return value in subregion represented by this node.
- *
- * @param v Vector.
- * @return Value in subregion represented by this node.
- */
- @Override public double process(Vector v) {
- return val;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/SplitNode.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/SplitNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/SplitNode.java
deleted file mode 100644
index 4c258d1..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/SplitNode.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.nodes;
-
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Node in decision tree representing a split.
- */
-public abstract class SplitNode implements DecisionTreeNode {
- /** Left subtree. */
- protected DecisionTreeNode l;
-
- /** Right subtree. */
- protected DecisionTreeNode r;
-
- /** Feature index. */
- protected final int featureIdx;
-
- /**
- * Constructs SplitNode with a given feature index.
- *
- * @param featureIdx Feature index.
- */
- public SplitNode(int featureIdx) {
- this.featureIdx = featureIdx;
- }
-
- /**
- * Indicates if the given vector is in left subtree.
- *
- * @param v Vector
- * @return Status of given vector being left subtree.
- */
- abstract boolean goLeft(Vector v);
-
- /**
- * Left subtree.
- *
- * @return Left subtree.
- */
- public DecisionTreeNode left() {
- return l;
- }
-
- /**
- * Right subtree.
- *
- * @return Right subtree.
- */
- public DecisionTreeNode right() {
- return r;
- }
-
- /**
- * Set the left subtree.
- *
- * @param n left subtree.
- */
- public void setLeft(DecisionTreeNode n) {
- l = n;
- }
-
- /**
- * Set the right subtree.
- *
- * @param n right subtree.
- */
- public void setRight(DecisionTreeNode n) {
- r = n;
- }
-
- /**
- * Delegates processing to subtrees.
- *
- * @param v Vector.
- * @return Value assigned to the given vector.
- */
- @Override public double process(Vector v) {
- if (left() != null && goLeft(v))
- return left().process(v);
- else
- return right().process(v);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/package-info.java
deleted file mode 100644
index d6deb9d..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains classes representing decision tree nodes.
- */
-package org.apache.ignite.ml.trees.nodes;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/package-info.java
deleted file mode 100644
index b07ba4a..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains decision tree algorithms.
- */
-package org.apache.ignite.ml.trees;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndex.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndex.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndex.java
deleted file mode 100644
index 0d27c8a..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndex.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-
-/**
- * Class representing a simple index in 2d matrix in the form (row, col).
- */
-public class BiIndex implements Externalizable {
- /** Row. */
- private int row;
-
- /** Column. */
- @AffinityKeyMapped
- private int col;
-
- /**
- * No-op constructor for serialization/deserialization.
- */
- public BiIndex() {
- // No-op.
- }
-
- /**
- * Construct BiIndex from row and column.
- *
- * @param row Row.
- * @param col Column.
- */
- public BiIndex(int row, int col) {
- this.row = row;
- this.col = col;
- }
-
- /**
- * Returns row.
- *
- * @return Row.
- */
- public int row() {
- return row;
- }
-
- /**
- * Returns column.
- *
- * @return Column.
- */
- public int col() {
- return col;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- BiIndex idx = (BiIndex)o;
-
- if (row != idx.row)
- return false;
- return col == idx.col;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = row;
- res = 31 * res + col;
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "BiIndex [" +
- "row=" + row +
- ", col=" + col +
- ']';
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeInt(row);
- out.writeInt(col);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- row = in.readInt();
- col = in.readInt();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndexedCacheColumnDecisionTreeTrainerInput.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndexedCacheColumnDecisionTreeTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndexedCacheColumnDecisionTreeTrainerInput.java
deleted file mode 100644
index 04281fb..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndexedCacheColumnDecisionTreeTrainerInput.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.util.Map;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.lang.IgniteBiTuple;
-
-/**
- * Adapter for column decision tree trainer for bi-indexed cache.
- */
-public class BiIndexedCacheColumnDecisionTreeTrainerInput extends CacheColumnDecisionTreeTrainerInput<BiIndex, Double> {
- /**
- * Construct an input for {@link ColumnDecisionTreeTrainer}.
- *
- * @param cache Bi-indexed cache.
- * @param catFeaturesInfo Information about categorical feature in the form (feature index -> number of
- * categories).
- * @param samplesCnt Count of samples.
- * @param featuresCnt Count of features.
- */
- public BiIndexedCacheColumnDecisionTreeTrainerInput(IgniteCache<BiIndex, Double> cache,
- Map<Integer, Integer> catFeaturesInfo, int samplesCnt, int featuresCnt) {
- super(cache,
- () -> IntStream.range(0, samplesCnt).mapToObj(s -> new BiIndex(s, featuresCnt)),
- e -> Stream.of(new IgniteBiTuple<>(e.getKey().row(), e.getValue())),
- DoubleStream::of,
- fIdx -> IntStream.range(0, samplesCnt).mapToObj(s -> new BiIndex(s, fIdx)),
- catFeaturesInfo,
- featuresCnt,
- samplesCnt);
- }
-
- /** {@inheritDoc} */
- @Override public Object affinityKey(int idx, Ignite ignite) {
- return idx;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/CacheColumnDecisionTreeTrainerInput.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/CacheColumnDecisionTreeTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/CacheColumnDecisionTreeTrainerInput.java
deleted file mode 100644
index 40927b7..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/CacheColumnDecisionTreeTrainerInput.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.Stream;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-
-/**
- * Adapter of a given cache to {@link CacheColumnDecisionTreeTrainerInput}
- *
- * @param <K> Class of keys of the cache.
- * @param <V> Class of values of the cache.
- */
-public abstract class CacheColumnDecisionTreeTrainerInput<K, V> implements ColumnDecisionTreeTrainerInput {
- /** Supplier of labels key. */
- private final IgniteSupplier<Stream<K>> labelsKeys;
-
- /** Count of features. */
- private final int featuresCnt;
-
- /** Function which maps feature index to Stream of keys corresponding to this feature index. */
- private final IgniteFunction<Integer, Stream<K>> keyMapper;
-
- /** Information about which features are categorical in form of feature index -> number of categories. */
- private final Map<Integer, Integer> catFeaturesInfo;
-
- /** Cache name. */
- private final String cacheName;
-
- /** Count of samples. */
- private final int samplesCnt;
-
- /** Function used for mapping cache values to stream of tuples. */
- private final IgniteFunction<Cache.Entry<K, V>, Stream<IgniteBiTuple<Integer, Double>>> valuesMapper;
-
- /**
- * Function which map value of entry with label key to DoubleStream.
- * Look at {@code CacheColumnDecisionTreeTrainerInput::labels} for understanding how {@code labelsKeys} and
- * {@code labelsMapper} interact.
- */
- private final IgniteFunction<V, DoubleStream> labelsMapper;
-
- /**
- * Constructs input for {@link ColumnDecisionTreeTrainer}.
- *
- * @param c Cache.
- * @param valuesMapper Function for mapping cache entry to stream used by {@link ColumnDecisionTreeTrainer}.
- * @param labelsMapper Function used for mapping cache value to labels array.
- * @param keyMapper Function used for mapping feature index to the cache key.
- * @param catFeaturesInfo Information about which features are categorical in form of feature index -> number of
- * categories.
- * @param featuresCnt Count of features.
- * @param samplesCnt Count of samples.
- */
- // TODO: IGNITE-5724 think about boxing/unboxing
- public CacheColumnDecisionTreeTrainerInput(IgniteCache<K, V> c,
- IgniteSupplier<Stream<K>> labelsKeys,
- IgniteFunction<Cache.Entry<K, V>, Stream<IgniteBiTuple<Integer, Double>>> valuesMapper,
- IgniteFunction<V, DoubleStream> labelsMapper,
- IgniteFunction<Integer, Stream<K>> keyMapper,
- Map<Integer, Integer> catFeaturesInfo,
- int featuresCnt, int samplesCnt) {
-
- cacheName = c.getName();
- this.labelsKeys = labelsKeys;
- this.valuesMapper = valuesMapper;
- this.labelsMapper = labelsMapper;
- this.keyMapper = keyMapper;
- this.catFeaturesInfo = catFeaturesInfo;
- this.samplesCnt = samplesCnt;
- this.featuresCnt = featuresCnt;
- }
-
- /** {@inheritDoc} */
- @Override public Stream<IgniteBiTuple<Integer, Double>> values(int idx) {
- return cache(Ignition.localIgnite()).getAll(keyMapper.apply(idx).collect(Collectors.toSet())).
- entrySet().
- stream().
- flatMap(ent -> valuesMapper.apply(new CacheEntryImpl<>(ent.getKey(), ent.getValue())));
- }
-
- /** {@inheritDoc} */
- @Override public double[] labels(Ignite ignite) {
- return labelsKeys.get().map(k -> get(k, ignite)).flatMapToDouble(labelsMapper).toArray();
- }
-
- /** {@inheritDoc} */
- @Override public Map<Integer, Integer> catFeaturesInfo() {
- return catFeaturesInfo;
- }
-
- /** {@inheritDoc} */
- @Override public int featuresCount() {
- return featuresCnt;
- }
-
- /** {@inheritDoc} */
- @Override public Object affinityKey(int idx, Ignite ignite) {
- return ignite.affinity(cacheName).affinityKey(keyMapper.apply(idx));
- }
-
- /** */
- private V get(K k, Ignite ignite) {
- V res = cache(ignite).localPeek(k);
-
- if (res == null)
- res = cache(ignite).get(k);
-
- return res;
- }
-
- /** */
- private IgniteCache<K, V> cache(Ignite ignite) {
- return ignite.getOrCreateCache(cacheName);
- }
-}
[04/14] ignite git commit: IGNITE-8059: Integrate decision tree with
partition based dataset.
Posted by ak...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainer.java
deleted file mode 100644
index fec0a83..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainer.java
+++ /dev/null
@@ -1,568 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CachePeekMode;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distributed.CacheUtils;
-import org.apache.ignite.ml.math.functions.Functions;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteCurriedBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.nodes.DecisionTreeNode;
-import org.apache.ignite.ml.trees.nodes.Leaf;
-import org.apache.ignite.ml.trees.nodes.SplitNode;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ContextCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache.FeatureKey;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ProjectionsCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ProjectionsCache.RegionKey;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.SplitCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.SplitCache.SplitKey;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.FeatureProcessor;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-import org.jetbrains.annotations.NotNull;
-
-import static org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache.getFeatureCacheKey;
-
-/**
- * This trainer stores observations as columns and features as rows.
- * Ideas from https://github.com/fabuzaid21/yggdrasil are used here.
- */
-public class ColumnDecisionTreeTrainer<D extends ContinuousRegionInfo> implements
- Trainer<DecisionTreeModel, ColumnDecisionTreeTrainerInput> {
- /**
- * Function used to assign a value to a region.
- */
- private final IgniteFunction<DoubleStream, Double> regCalc;
-
- /**
- * Function used to calculate impurity in regions used by categorical features.
- */
- private final IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> continuousCalculatorProvider;
-
- /**
- * Categorical calculator provider.
- **/
- private final IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> categoricalCalculatorProvider;
-
- /**
- * Cache used for storing data for training.
- */
- private IgniteCache<RegionKey, List<RegionProjection>> prjsCache;
-
- /**
- * Minimal information gain.
- */
- private static final double MIN_INFO_GAIN = 1E-10;
-
- /**
- * Maximal depth of the decision tree.
- */
- private final int maxDepth;
-
- /**
- * Size of block which is used for storing regions in cache.
- */
- private static final int BLOCK_SIZE = 1 << 4;
-
- /** Ignite instance. */
- private final Ignite ignite;
-
- /** Logger */
- private final IgniteLogger log;
-
- /**
- * Construct {@link ColumnDecisionTreeTrainer}.
- *
- * @param maxDepth Maximal depth of the decision tree.
- * @param continuousCalculatorProvider Provider of calculator of splits for region projection on continuous
- * features.
- * @param categoricalCalculatorProvider Provider of calculator of splits for region projection on categorical
- * features.
- * @param regCalc Function used to assign a value to a region.
- */
- public ColumnDecisionTreeTrainer(int maxDepth,
- IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> continuousCalculatorProvider,
- IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> categoricalCalculatorProvider,
- IgniteFunction<DoubleStream, Double> regCalc,
- Ignite ignite) {
- this.maxDepth = maxDepth;
- this.continuousCalculatorProvider = continuousCalculatorProvider;
- this.categoricalCalculatorProvider = categoricalCalculatorProvider;
- this.regCalc = regCalc;
- this.ignite = ignite;
- this.log = ignite.log();
- }
-
- /**
- * Utility class used to get index of feature by which split is done and split info.
- */
- private static class IndexAndSplitInfo {
- /**
- * Index of feature by which split is done.
- */
- private final int featureIdx;
-
- /**
- * Split information.
- */
- private final SplitInfo info;
-
- /**
- * @param featureIdx Index of feature by which split is done.
- * @param info Split information.
- */
- IndexAndSplitInfo(int featureIdx, SplitInfo info) {
- this.featureIdx = featureIdx;
- this.info = info;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "IndexAndSplitInfo [featureIdx=" + featureIdx + ", info=" + info + ']';
- }
- }
-
- /**
- * Utility class used to build decision tree. Basically it is pointer to leaf node.
- */
- private static class TreeTip {
- /** */
- private Consumer<DecisionTreeNode> leafSetter;
-
- /** */
- private int depth;
-
- /** */
- TreeTip(Consumer<DecisionTreeNode> leafSetter, int depth) {
- this.leafSetter = leafSetter;
- this.depth = depth;
- }
- }
-
- /**
- * Utility class used as decision tree root node.
- */
- private static class RootNode implements DecisionTreeNode {
- /** */
- private DecisionTreeNode s;
-
- /**
- * {@inheritDoc}
- */
- @Override public double process(Vector v) {
- return s.process(v);
- }
-
- /** */
- void setSplit(DecisionTreeNode s) {
- this.s = s;
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public DecisionTreeModel train(ColumnDecisionTreeTrainerInput i) {
- prjsCache = ProjectionsCache.getOrCreate(ignite);
- IgniteCache<UUID, TrainingContext<D>> ctxtCache = ContextCache.getOrCreate(ignite);
- SplitCache.getOrCreate(ignite);
-
- UUID trainingUUID = UUID.randomUUID();
-
- TrainingContext<D> ct = new TrainingContext<>(i, continuousCalculatorProvider.apply(i), categoricalCalculatorProvider.apply(i), trainingUUID, ignite);
- ctxtCache.put(trainingUUID, ct);
-
- CacheUtils.bcast(prjsCache.getName(), ignite, () -> {
- Ignite ignite = Ignition.localIgnite();
- IgniteCache<RegionKey, List<RegionProjection>> projCache = ProjectionsCache.getOrCreate(ignite);
- IgniteCache<FeatureKey, double[]> featuresCache = FeaturesCache.getOrCreate(ignite);
-
- Affinity<RegionKey> targetAffinity = ignite.affinity(ProjectionsCache.CACHE_NAME);
-
- ClusterNode locNode = ignite.cluster().localNode();
-
- Map<FeatureKey, double[]> fm = new ConcurrentHashMap<>();
- Map<RegionKey, List<RegionProjection>> pm = new ConcurrentHashMap<>();
-
- targetAffinity.
- mapKeysToNodes(IntStream.range(0, i.featuresCount()).
- mapToObj(idx -> ProjectionsCache.key(idx, 0, i.affinityKey(idx, ignite), trainingUUID)).
- collect(Collectors.toSet())).getOrDefault(locNode, Collections.emptyList()).
- forEach(k -> {
- FeatureProcessor vec;
-
- int featureIdx = k.featureIdx();
-
- IgniteCache<UUID, TrainingContext<D>> ctxCache = ContextCache.getOrCreate(ignite);
- TrainingContext ctx = ctxCache.get(trainingUUID);
- double[] vals = new double[ctx.labels().length];
-
- vec = ctx.featureProcessor(featureIdx);
- i.values(featureIdx).forEach(t -> vals[t.get1()] = t.get2());
-
- fm.put(getFeatureCacheKey(featureIdx, trainingUUID, i.affinityKey(featureIdx, ignite)), vals);
-
- List<RegionProjection> newReg = new ArrayList<>(BLOCK_SIZE);
- newReg.add(vec.createInitialRegion(getSamples(i.values(featureIdx), ctx.labels().length), vals, ctx.labels()));
- pm.put(k, newReg);
- });
-
- featuresCache.putAll(fm);
- projCache.putAll(pm);
-
- return null;
- });
-
- return doTrain(i, trainingUUID);
- }
-
- /**
- * Get samples array.
- *
- * @param values Stream of tuples in the form of (index, value).
- * @param size size of stream.
- * @return Samples array.
- */
- private Integer[] getSamples(Stream<IgniteBiTuple<Integer, Double>> values, int size) {
- Integer[] res = new Integer[size];
-
- values.forEach(v -> res[v.get1()] = v.get1());
-
- return res;
- }
-
- /** */
- @NotNull
- private DecisionTreeModel doTrain(ColumnDecisionTreeTrainerInput input, UUID uuid) {
- RootNode root = new RootNode();
-
- // List containing setters of leaves of the tree.
- List<TreeTip> tips = new LinkedList<>();
- tips.add(new TreeTip(root::setSplit, 0));
-
- int curDepth = 0;
- int regsCnt = 1;
-
- int featuresCnt = input.featuresCount();
- IntStream.range(0, featuresCnt).mapToObj(fIdx -> SplitCache.key(fIdx, input.affinityKey(fIdx, ignite), uuid)).
- forEach(k -> SplitCache.getOrCreate(ignite).put(k, new IgniteBiTuple<>(0, 0.0)));
- updateSplitCache(0, regsCnt, featuresCnt, ig -> i -> input.affinityKey(i, ig), uuid);
-
- // TODO: IGNITE-5893 Currently if the best split makes tree deeper than max depth process will be terminated, but actually we should
- // only stop when *any* improving split makes tree deeper than max depth. Can be fixed if we will store which
- // regions cannot be split more and split only those that can.
- while (true) {
- long before = System.currentTimeMillis();
-
- IgniteBiTuple<Integer, IgniteBiTuple<Integer, Double>> b = findBestSplitIndexForFeatures(featuresCnt, input::affinityKey, uuid);
-
- long findBestRegIdx = System.currentTimeMillis() - before;
-
- Integer bestFeatureIdx = b.get1();
-
- Integer regIdx = b.get2().get1();
- Double bestInfoGain = b.get2().get2();
-
- if (regIdx >= 0 && bestInfoGain > MIN_INFO_GAIN) {
- before = System.currentTimeMillis();
-
- SplitInfo bi = ignite.compute().affinityCall(ProjectionsCache.CACHE_NAME,
- input.affinityKey(bestFeatureIdx, ignite),
- () -> {
- TrainingContext<ContinuousRegionInfo> ctx = ContextCache.getOrCreate(ignite).get(uuid);
- Ignite ignite = Ignition.localIgnite();
- RegionKey key = ProjectionsCache.key(bestFeatureIdx,
- regIdx / BLOCK_SIZE,
- input.affinityKey(bestFeatureIdx, Ignition.localIgnite()),
- uuid);
- RegionProjection reg = ProjectionsCache.getOrCreate(ignite).localPeek(key).get(regIdx % BLOCK_SIZE);
- return ctx.featureProcessor(bestFeatureIdx).findBestSplit(reg, ctx.values(bestFeatureIdx, ignite), ctx.labels(), regIdx);
- });
-
- long findBestSplit = System.currentTimeMillis() - before;
-
- IndexAndSplitInfo best = new IndexAndSplitInfo(bestFeatureIdx, bi);
-
- regsCnt++;
-
- if (log.isDebugEnabled())
- log.debug("Globally best: " + best.info + " idx time: " + findBestRegIdx + ", calculate best: " + findBestSplit + " fi: " + best.featureIdx + ", regs: " + regsCnt);
- // Request bitset for split region.
- int ind = best.info.regionIndex();
-
- SparseBitSet bs = ignite.compute().affinityCall(ProjectionsCache.CACHE_NAME,
- input.affinityKey(bestFeatureIdx, ignite),
- () -> {
- Ignite ignite = Ignition.localIgnite();
- IgniteCache<FeatureKey, double[]> featuresCache = FeaturesCache.getOrCreate(ignite);
- IgniteCache<UUID, TrainingContext<D>> ctxCache = ContextCache.getOrCreate(ignite);
- TrainingContext ctx = ctxCache.localPeek(uuid);
-
- double[] values = featuresCache.localPeek(getFeatureCacheKey(bestFeatureIdx, uuid, input.affinityKey(bestFeatureIdx, Ignition.localIgnite())));
- RegionKey key = ProjectionsCache.key(bestFeatureIdx,
- regIdx / BLOCK_SIZE,
- input.affinityKey(bestFeatureIdx, Ignition.localIgnite()),
- uuid);
- RegionProjection reg = ProjectionsCache.getOrCreate(ignite).localPeek(key).get(regIdx % BLOCK_SIZE);
- return ctx.featureProcessor(bestFeatureIdx).calculateOwnershipBitSet(reg, values, best.info);
-
- });
-
- SplitNode sn = best.info.createSplitNode(best.featureIdx);
-
- TreeTip tipToSplit = tips.get(ind);
- tipToSplit.leafSetter.accept(sn);
- tipToSplit.leafSetter = sn::setLeft;
- int d = tipToSplit.depth++;
- tips.add(new TreeTip(sn::setRight, d));
-
- if (d > curDepth) {
- curDepth = d;
- if (log.isDebugEnabled()) {
- log.debug("Depth: " + curDepth);
- log.debug("Cache size: " + prjsCache.size(CachePeekMode.PRIMARY));
- }
- }
-
- before = System.currentTimeMillis();
- // Perform split on all feature vectors.
- IgniteSupplier<Set<RegionKey>> bestRegsKeys = () -> IntStream.range(0, featuresCnt).
- mapToObj(fIdx -> ProjectionsCache.key(fIdx, ind / BLOCK_SIZE, input.affinityKey(fIdx, Ignition.localIgnite()), uuid)).
- collect(Collectors.toSet());
-
- int rc = regsCnt;
-
- // Perform split.
- CacheUtils.update(prjsCache.getName(), ignite,
- (Ignite ign, Cache.Entry<RegionKey, List<RegionProjection>> e) -> {
- RegionKey k = e.getKey();
-
- List<RegionProjection> leftBlock = e.getValue();
-
- int fIdx = k.featureIdx();
- int idxInBlock = ind % BLOCK_SIZE;
-
- IgniteCache<UUID, TrainingContext<D>> ctxCache = ContextCache.getOrCreate(ign);
- TrainingContext<D> ctx = ctxCache.get(uuid);
-
- RegionProjection targetRegProj = leftBlock.get(idxInBlock);
-
- IgniteBiTuple<RegionProjection, RegionProjection> regs = ctx.
- performSplit(input, bs, fIdx, best.featureIdx, targetRegProj, best.info.leftData(), best.info.rightData(), ign);
-
- RegionProjection left = regs.get1();
- RegionProjection right = regs.get2();
-
- leftBlock.set(idxInBlock, left);
- RegionKey rightKey = ProjectionsCache.key(fIdx, (rc - 1) / BLOCK_SIZE, input.affinityKey(fIdx, ign), uuid);
-
- IgniteCache<RegionKey, List<RegionProjection>> c = ProjectionsCache.getOrCreate(ign);
-
- List<RegionProjection> rightBlock = rightKey.equals(k) ? leftBlock : c.localPeek(rightKey);
-
- if (rightBlock == null) {
- List<RegionProjection> newBlock = new ArrayList<>(BLOCK_SIZE);
- newBlock.add(right);
- return Stream.of(new CacheEntryImpl<>(k, leftBlock), new CacheEntryImpl<>(rightKey, newBlock));
- }
- else {
- rightBlock.add(right);
- return rightBlock.equals(k) ?
- Stream.of(new CacheEntryImpl<>(k, leftBlock)) :
- Stream.of(new CacheEntryImpl<>(k, leftBlock), new CacheEntryImpl<>(rightKey, rightBlock));
- }
- },
- bestRegsKeys);
-
- if (log.isDebugEnabled())
- log.debug("Update of projections cache time: " + (System.currentTimeMillis() - before));
-
- before = System.currentTimeMillis();
-
- updateSplitCache(ind, rc, featuresCnt, ig -> i -> input.affinityKey(i, ig), uuid);
-
- if (log.isDebugEnabled())
- log.debug("Update of split cache time: " + (System.currentTimeMillis() - before));
- }
- else {
- if (log.isDebugEnabled())
- log.debug("Best split [bestFeatureIdx=" + bestFeatureIdx + ", bestInfoGain=" + bestInfoGain + "]");
- break;
- }
- }
-
- int rc = regsCnt;
-
- IgniteSupplier<Iterable<Cache.Entry<RegionKey, List<RegionProjection>>>> featZeroRegs = () -> {
- IgniteCache<RegionKey, List<RegionProjection>> projsCache = ProjectionsCache.getOrCreate(Ignition.localIgnite());
-
- return () -> IntStream.range(0, (rc - 1) / BLOCK_SIZE + 1).
- mapToObj(rBIdx -> ProjectionsCache.key(0, rBIdx, input.affinityKey(0, Ignition.localIgnite()), uuid)).
- map(k -> (Cache.Entry<RegionKey, List<RegionProjection>>)new CacheEntryImpl<>(k, projsCache.localPeek(k))).iterator();
- };
-
- Map<Integer, Double> vals = CacheUtils.reduce(prjsCache.getName(), ignite,
- (TrainingContext ctx, Cache.Entry<RegionKey, List<RegionProjection>> e, Map<Integer, Double> m) -> {
- int regBlockIdx = e.getKey().regionBlockIndex();
-
- if (e.getValue() != null) {
- for (int i = 0; i < e.getValue().size(); i++) {
- int regIdx = regBlockIdx * BLOCK_SIZE + i;
- RegionProjection reg = e.getValue().get(i);
-
- Double res = regCalc.apply(Arrays.stream(reg.sampleIndexes()).mapToDouble(s -> ctx.labels()[s]));
- m.put(regIdx, res);
- }
- }
-
- return m;
- },
- () -> ContextCache.getOrCreate(Ignition.localIgnite()).get(uuid),
- featZeroRegs,
- (infos, infos2) -> {
- Map<Integer, Double> res = new HashMap<>();
- res.putAll(infos);
- res.putAll(infos2);
- return res;
- },
- HashMap::new
- );
-
- int i = 0;
- for (TreeTip tip : tips) {
- tip.leafSetter.accept(new Leaf(vals.get(i)));
- i++;
- }
-
- ProjectionsCache.clear(featuresCnt, rc, input::affinityKey, uuid, ignite);
- ContextCache.getOrCreate(ignite).remove(uuid);
- FeaturesCache.clear(featuresCnt, input::affinityKey, uuid, ignite);
- SplitCache.clear(featuresCnt, input::affinityKey, uuid, ignite);
-
- return new DecisionTreeModel(root.s);
- }
-
- /**
- * Find the best split in the form (feature index, (index of region with the best split, impurity of region with the
- * best split)).
- *
- * @param featuresCnt Count of features.
- * @param affinity Affinity function.
- * @param trainingUUID UUID of training.
- * @return Best split in the form (feature index, (index of region with the best split, impurity of region with the
- * best split)).
- */
- private IgniteBiTuple<Integer, IgniteBiTuple<Integer, Double>> findBestSplitIndexForFeatures(int featuresCnt,
- IgniteBiFunction<Integer, Ignite, Object> affinity,
- UUID trainingUUID) {
- Set<Integer> featureIndexes = IntStream.range(0, featuresCnt).boxed().collect(Collectors.toSet());
-
- return CacheUtils.reduce(SplitCache.CACHE_NAME, ignite,
- (Object ctx, Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>> e, IgniteBiTuple<Integer, IgniteBiTuple<Integer, Double>> r) ->
- Functions.MAX_GENERIC(new IgniteBiTuple<>(e.getKey().featureIdx(), e.getValue()), r, comparator()),
- () -> null,
- () -> SplitCache.localEntries(featureIndexes, affinity, trainingUUID),
- (i1, i2) -> Functions.MAX_GENERIC(i1, i2, Comparator.comparingDouble(bt -> bt.get2().get2())),
- () -> new IgniteBiTuple<>(-1, new IgniteBiTuple<>(-1, Double.NEGATIVE_INFINITY))
- );
- }
-
- /** */
- private static Comparator<IgniteBiTuple<Integer, IgniteBiTuple<Integer, Double>>> comparator() {
- return Comparator.comparingDouble(bt -> bt != null && bt.get2() != null ? bt.get2().get2() : Double.NEGATIVE_INFINITY);
- }
-
- /**
- * Update split cache.
- *
- * @param lastSplitRegionIdx Index of region which had last best split.
- * @param regsCnt Count of regions.
- * @param featuresCnt Count of features.
- * @param affinity Affinity function.
- * @param trainingUUID UUID of current training.
- */
- private void updateSplitCache(int lastSplitRegionIdx, int regsCnt, int featuresCnt,
- IgniteCurriedBiFunction<Ignite, Integer, Object> affinity,
- UUID trainingUUID) {
- CacheUtils.update(SplitCache.CACHE_NAME, ignite,
- (Ignite ign, Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>> e) -> {
- Integer bestRegIdx = e.getValue().get1();
- int fIdx = e.getKey().featureIdx();
- TrainingContext ctx = ContextCache.getOrCreate(ign).get(trainingUUID);
-
- Map<Integer, RegionProjection> toCompare;
-
- // Fully recalculate best.
- if (bestRegIdx == lastSplitRegionIdx)
- toCompare = ProjectionsCache.projectionsOfFeature(fIdx, maxDepth, regsCnt, BLOCK_SIZE, affinity.apply(ign), trainingUUID, ign);
- // Just compare previous best and two regions which are produced by split.
- else
- toCompare = ProjectionsCache.projectionsOfRegions(fIdx, maxDepth,
- IntStream.of(bestRegIdx, lastSplitRegionIdx, regsCnt - 1), BLOCK_SIZE, affinity.apply(ign), trainingUUID, ign);
-
- double[] values = ctx.values(fIdx, ign);
- double[] labels = ctx.labels();
-
- Optional<IgniteBiTuple<Integer, Double>> max = toCompare.entrySet().stream().
- map(ent -> {
- SplitInfo bestSplit = ctx.featureProcessor(fIdx).findBestSplit(ent.getValue(), values, labels, ent.getKey());
- return new IgniteBiTuple<>(ent.getKey(), bestSplit != null ? bestSplit.infoGain() : Double.NEGATIVE_INFINITY);
- }).
- max(Comparator.comparingDouble(IgniteBiTuple::get2));
-
- return max.<Stream<Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>>>>
- map(objects -> Stream.of(new CacheEntryImpl<>(e.getKey(), objects))).orElseGet(Stream::empty);
- },
- () -> IntStream.range(0, featuresCnt).mapToObj(fIdx -> SplitCache.key(fIdx, affinity.apply(ignite).apply(fIdx), trainingUUID)).collect(Collectors.toSet())
- );
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainerInput.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainerInput.java
deleted file mode 100644
index bf8790b..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainerInput.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.util.Map;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.lang.IgniteBiTuple;
-
-/**
- * Input for {@link ColumnDecisionTreeTrainer}.
- */
-public interface ColumnDecisionTreeTrainerInput {
- /**
- * Projection of data on feature with the given index.
- *
- * @param idx Feature index.
- * @return Projection of data on feature with the given index.
- */
- Stream<IgniteBiTuple<Integer, Double>> values(int idx);
-
- /**
- * Labels.
- *
- * @param ignite Ignite instance.
- */
- double[] labels(Ignite ignite);
-
- /** Information about which features are categorical in the form of feature index -> number of categories. */
- Map<Integer, Integer> catFeaturesInfo();
-
- /** Number of features. */
- int featuresCount();
-
- /**
- * Get affinity key for the given column index.
- * Affinity key should be pure-functionally dependent from idx.
- */
- Object affinityKey(int idx, Ignite ignite);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/MatrixColumnDecisionTreeTrainerInput.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/MatrixColumnDecisionTreeTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/MatrixColumnDecisionTreeTrainerInput.java
deleted file mode 100644
index 3da6bad..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/MatrixColumnDecisionTreeTrainerInput.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey;
-import org.apache.ignite.ml.math.distributed.keys.impl.SparseMatrixKey;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * Adapter of SparseDistributedMatrix to ColumnDecisionTreeTrainerInput.
- * Sparse SparseDistributedMatrix should be in {@link StorageConstants#COLUMN_STORAGE_MODE} and
- * should contain samples in rows last position in row being label of this sample.
- */
-public class MatrixColumnDecisionTreeTrainerInput extends CacheColumnDecisionTreeTrainerInput<RowColMatrixKey, Map<Integer, Double>> {
- /**
- * @param m Sparse SparseDistributedMatrix should be in {@link StorageConstants#COLUMN_STORAGE_MODE}
- * containing samples in rows last position in row being label of this sample.
- * @param catFeaturesInfo Information about which features are categorical in form of feature index -> number of
- * categories.
- */
- public MatrixColumnDecisionTreeTrainerInput(SparseDistributedMatrix m, Map<Integer, Integer> catFeaturesInfo) {
- super(((SparseDistributedMatrixStorage)m.getStorage()).cache(),
- () -> Stream.of(new SparseMatrixKey(m.columnSize() - 1, m.getUUID(), m.columnSize() - 1)),
- valuesMapper(m),
- labels(m),
- keyMapper(m),
- catFeaturesInfo,
- m.columnSize() - 1,
- m.rowSize());
- }
-
- /** Values mapper. See {@link CacheColumnDecisionTreeTrainerInput#valuesMapper} */
- @NotNull
- private static IgniteFunction<Cache.Entry<RowColMatrixKey, Map<Integer, Double>>, Stream<IgniteBiTuple<Integer, Double>>> valuesMapper(
- SparseDistributedMatrix m) {
- return ent -> {
- Map<Integer, Double> map = ent.getValue() != null ? ent.getValue() : new HashMap<>();
- return IntStream.range(0, m.rowSize()).mapToObj(k -> new IgniteBiTuple<>(k, map.getOrDefault(k, 0.0)));
- };
- }
-
- /** Key mapper. See {@link CacheColumnDecisionTreeTrainerInput#keyMapper} */
- @NotNull private static IgniteFunction<Integer, Stream<RowColMatrixKey>> keyMapper(SparseDistributedMatrix m) {
- return i -> Stream.of(new SparseMatrixKey(i, ((SparseDistributedMatrixStorage)m.getStorage()).getUUID(), i));
- }
-
- /** Labels mapper. See {@link CacheColumnDecisionTreeTrainerInput#labelsMapper} */
- @NotNull private static IgniteFunction<Map<Integer, Double>, DoubleStream> labels(SparseDistributedMatrix m) {
- return mp -> IntStream.range(0, m.rowSize()).mapToDouble(k -> mp.getOrDefault(k, 0.0));
- }
-
- /** {@inheritDoc} */
- @Override public Object affinityKey(int idx, Ignite ignite) {
- return idx;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/RegionProjection.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/RegionProjection.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/RegionProjection.java
deleted file mode 100644
index e95f57b..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/RegionProjection.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import org.apache.ignite.ml.trees.RegionInfo;
-
-/**
- * Projection of region on given feature.
- *
- * @param <D> Data of region.
- */
-public class RegionProjection<D extends RegionInfo> implements Externalizable {
- /** Samples projections. */
- protected Integer[] sampleIndexes;
-
- /** Region data */
- protected D data;
-
- /** Depth of this region. */
- protected int depth;
-
- /**
- * @param sampleIndexes Samples indexes.
- * @param data Region data.
- * @param depth Depth of this region.
- */
- public RegionProjection(Integer[] sampleIndexes, D data, int depth) {
- this.data = data;
- this.depth = depth;
- this.sampleIndexes = sampleIndexes;
- }
-
- /**
- * No-op constructor used for serialization/deserialization.
- */
- public RegionProjection() {
- // No-op.
- }
-
- /**
- * Get samples indexes.
- *
- * @return Samples indexes.
- */
- public Integer[] sampleIndexes() {
- return sampleIndexes;
- }
-
- /**
- * Get region data.
- *
- * @return Region data.
- */
- public D data() {
- return data;
- }
-
- /**
- * Get region depth.
- *
- * @return Region depth.
- */
- public int depth() {
- return depth;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeInt(sampleIndexes.length);
-
- for (Integer sampleIndex : sampleIndexes)
- out.writeInt(sampleIndex);
-
- out.writeObject(data);
- out.writeInt(depth);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- int size = in.readInt();
-
- sampleIndexes = new Integer[size];
-
- for (int i = 0; i < size; i++)
- sampleIndexes[i] = in.readInt();
-
- data = (D)in.readObject();
- depth = in.readInt();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/TrainingContext.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/TrainingContext.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/TrainingContext.java
deleted file mode 100644
index 6415dab..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/TrainingContext.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import java.util.Map;
-import java.util.UUID;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.CategoricalFeatureProcessor;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.ContinuousFeatureProcessor;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.FeatureProcessor;
-
-import static org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache.COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME;
-
-/**
- * Context of training with {@link ColumnDecisionTreeTrainer}.
- *
- * @param <D> Class for storing of information used in calculation of impurity of continuous feature region.
- */
-public class TrainingContext<D extends ContinuousRegionInfo> {
- /** Input for training with {@link ColumnDecisionTreeTrainer}. */
- private final ColumnDecisionTreeTrainerInput input;
-
- /** Labels. */
- private final double[] labels;
-
- /** Calculator used for finding splits of region of continuous features. */
- private final ContinuousSplitCalculator<D> continuousSplitCalculator;
-
- /** Calculator used for finding splits of region of categorical feature. */
- private final IgniteFunction<DoubleStream, Double> categoricalSplitCalculator;
-
- /** UUID of current training. */
- private final UUID trainingUUID;
-
- /**
- * Construct context for training with {@link ColumnDecisionTreeTrainer}.
- *
- * @param input Input for training.
- * @param continuousSplitCalculator Calculator used for calculations of splits of continuous features regions.
- * @param categoricalSplitCalculator Calculator used for calculations of splits of categorical features regions.
- * @param trainingUUID UUID of the current training.
- * @param ignite Ignite instance.
- */
- public TrainingContext(ColumnDecisionTreeTrainerInput input,
- ContinuousSplitCalculator<D> continuousSplitCalculator,
- IgniteFunction<DoubleStream, Double> categoricalSplitCalculator,
- UUID trainingUUID,
- Ignite ignite) {
- this.input = input;
- this.labels = input.labels(ignite);
- this.continuousSplitCalculator = continuousSplitCalculator;
- this.categoricalSplitCalculator = categoricalSplitCalculator;
- this.trainingUUID = trainingUUID;
- }
-
- /**
- * Get processor used for calculating splits of categorical features.
- *
- * @param catsCnt Count of categories.
- * @return Processor used for calculating splits of categorical features.
- */
- public CategoricalFeatureProcessor categoricalFeatureProcessor(int catsCnt) {
- return new CategoricalFeatureProcessor(categoricalSplitCalculator, catsCnt);
- }
-
- /**
- * Get processor used for calculating splits of continuous features.
- *
- * @return Processor used for calculating splits of continuous features.
- */
- public ContinuousFeatureProcessor<D> continuousFeatureProcessor() {
- return new ContinuousFeatureProcessor<>(continuousSplitCalculator);
- }
-
- /**
- * Get labels.
- *
- * @return Labels.
- */
- public double[] labels() {
- return labels;
- }
-
- /**
- * Get values of feature with given index.
- *
- * @param featIdx Feature index.
- * @param ignite Ignite instance.
- * @return Values of feature with given index.
- */
- public double[] values(int featIdx, Ignite ignite) {
- IgniteCache<FeaturesCache.FeatureKey, double[]> featuresCache = ignite.getOrCreateCache(COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME);
- return featuresCache.localPeek(FeaturesCache.getFeatureCacheKey(featIdx, trainingUUID, input.affinityKey(featIdx, ignite)));
- }
-
- /**
- * Perform best split on the given region projection.
- *
- * @param input Input of {@link ColumnDecisionTreeTrainer} performing split.
- * @param bitSet Bit set specifying split.
- * @param targetFeatIdx Index of feature for performing split.
- * @param bestFeatIdx Index of feature with best split.
- * @param targetRegionPrj Projection of region to split on feature with index {@code featureIdx}.
- * @param leftData Data of left region of split.
- * @param rightData Data of right region of split.
- * @param ignite Ignite instance.
- * @return Perform best split on the given region projection.
- */
- public IgniteBiTuple<RegionProjection, RegionProjection> performSplit(ColumnDecisionTreeTrainerInput input,
- SparseBitSet bitSet, int targetFeatIdx, int bestFeatIdx, RegionProjection targetRegionPrj, RegionInfo leftData,
- RegionInfo rightData, Ignite ignite) {
-
- Map<Integer, Integer> catFeaturesInfo = input.catFeaturesInfo();
-
- if (!catFeaturesInfo.containsKey(targetFeatIdx) && !catFeaturesInfo.containsKey(bestFeatIdx))
- return continuousFeatureProcessor().performSplit(bitSet, targetRegionPrj, (D)leftData, (D)rightData);
- else if (catFeaturesInfo.containsKey(targetFeatIdx))
- return categoricalFeatureProcessor(catFeaturesInfo.get(targetFeatIdx)).performSplitGeneric(bitSet, values(targetFeatIdx, ignite), targetRegionPrj, leftData, rightData);
- return continuousFeatureProcessor().performSplitGeneric(bitSet, labels, targetRegionPrj, leftData, rightData);
- }
-
- /**
- * Processor used for calculating splits for feature with the given index.
- *
- * @param featureIdx Index of feature to process.
- * @return Processor used for calculating splits for feature with the given index.
- */
- public FeatureProcessor featureProcessor(int featureIdx) {
- return input.catFeaturesInfo().containsKey(featureIdx) ? categoricalFeatureProcessor(input.catFeaturesInfo().get(featureIdx)) : continuousFeatureProcessor();
- }
-
- /**
- * Shortcut for affinity key.
- *
- * @param idx Feature index.
- * @return Affinity key.
- */
- public Object affinityKey(int idx) {
- return input.affinityKey(idx, Ignition.localIgnite());
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ContextCache.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ContextCache.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ContextCache.java
deleted file mode 100644
index 51ea359..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ContextCache.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
-
-import java.util.UUID;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.TrainingContext;
-
-/**
- * Class for operations related to cache containing training context for {@link ColumnDecisionTreeTrainer}.
- */
-public class ContextCache {
- /**
- * Name of cache containing training context for {@link ColumnDecisionTreeTrainer}.
- */
- public static final String COLUMN_DECISION_TREE_TRAINER_CONTEXT_CACHE_NAME = "COLUMN_DECISION_TREE_TRAINER_CONTEXT_CACHE_NAME";
-
- /**
- * Get or create cache for training context.
- *
- * @param ignite Ignite instance.
- * @param <D> Class storing information about continuous regions.
- * @return Cache for training context.
- */
- public static <D extends ContinuousRegionInfo> IgniteCache<UUID, TrainingContext<D>> getOrCreate(Ignite ignite) {
- CacheConfiguration<UUID, TrainingContext<D>> cfg = new CacheConfiguration<>();
-
- cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
-
- cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
- cfg.setEvictionPolicy(null);
-
- cfg.setCopyOnRead(false);
-
- cfg.setCacheMode(CacheMode.REPLICATED);
-
- cfg.setOnheapCacheEnabled(true);
-
- cfg.setReadFromBackup(true);
-
- cfg.setName(COLUMN_DECISION_TREE_TRAINER_CONTEXT_CACHE_NAME);
-
- return ignite.getOrCreateCache(cfg);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/FeaturesCache.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/FeaturesCache.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/FeaturesCache.java
deleted file mode 100644
index fcc1f16..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/FeaturesCache.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
-
-import java.util.Set;
-import java.util.UUID;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-
-/**
- * Cache storing features for {@link ColumnDecisionTreeTrainer}.
- */
-public class FeaturesCache {
- /**
- * Name of cache which is used for storing features for {@link ColumnDecisionTreeTrainer}.
- */
- public static final String COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME = "COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME";
-
- /**
- * Key of features cache.
- */
- public static class FeatureKey {
- /** Column key of cache used as input for {@link ColumnDecisionTreeTrainer}. */
- @AffinityKeyMapped
- private Object parentColKey;
-
- /** Index of feature. */
- private final int featureIdx;
-
- /** UUID of training. */
- private final UUID trainingUUID;
-
- /**
- * Construct FeatureKey.
- *
- * @param featureIdx Feature index.
- * @param trainingUUID UUID of training.
- * @param parentColKey Column key of cache used as input.
- */
- public FeatureKey(int featureIdx, UUID trainingUUID, Object parentColKey) {
- this.parentColKey = parentColKey;
- this.featureIdx = featureIdx;
- this.trainingUUID = trainingUUID;
- this.parentColKey = parentColKey;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- FeatureKey key = (FeatureKey)o;
-
- if (featureIdx != key.featureIdx)
- return false;
- return trainingUUID != null ? trainingUUID.equals(key.trainingUUID) : key.trainingUUID == null;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = trainingUUID != null ? trainingUUID.hashCode() : 0;
- res = 31 * res + featureIdx;
- return res;
- }
- }
-
- /**
- * Create new projections cache for ColumnDecisionTreeTrainer if needed.
- *
- * @param ignite Ignite instance.
- */
- public static IgniteCache<FeatureKey, double[]> getOrCreate(Ignite ignite) {
- CacheConfiguration<FeatureKey, double[]> cfg = new CacheConfiguration<>();
-
- // Write to primary.
- cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
- // Atomic transactions only.
- cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
- // No eviction.
- cfg.setEvictionPolicy(null);
-
- // No copying of values.
- cfg.setCopyOnRead(false);
-
- // Cache is partitioned.
- cfg.setCacheMode(CacheMode.PARTITIONED);
-
- cfg.setOnheapCacheEnabled(true);
-
- cfg.setBackups(0);
-
- cfg.setName(COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME);
-
- return ignite.getOrCreateCache(cfg);
- }
-
- /**
- * Construct FeatureKey from index, uuid and affinity key.
- *
- * @param idx Feature index.
- * @param uuid UUID of training.
- * @param aff Affinity key.
- * @return FeatureKey.
- */
- public static FeatureKey getFeatureCacheKey(int idx, UUID uuid, Object aff) {
- return new FeatureKey(idx, uuid, aff);
- }
-
- /**
- * Clear all data from features cache related to given training.
- *
- * @param featuresCnt Count of features.
- * @param affinity Affinity function.
- * @param uuid Training uuid.
- * @param ignite Ignite instance.
- */
- public static void clear(int featuresCnt, IgniteBiFunction<Integer, Ignite, Object> affinity, UUID uuid,
- Ignite ignite) {
- Set<FeatureKey> toRmv = IntStream.range(0, featuresCnt).boxed().map(fIdx -> getFeatureCacheKey(fIdx, uuid, affinity.apply(fIdx, ignite))).collect(Collectors.toSet());
-
- getOrCreate(ignite).removeAll(toRmv);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ProjectionsCache.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ProjectionsCache.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ProjectionsCache.java
deleted file mode 100644
index 080cb66..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ProjectionsCache.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.PrimitiveIterator;
-import java.util.Set;
-import java.util.UUID;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.RegionProjection;
-
-/**
- * Cache used for storing data of region projections on features.
- */
-public class ProjectionsCache {
- /**
- * Name of cache which is used for storing data of region projections on features of {@link
- * ColumnDecisionTreeTrainer}.
- */
- public static final String CACHE_NAME = "COLUMN_DECISION_TREE_TRAINER_PROJECTIONS_CACHE_NAME";
-
- /**
- * Key of region projections cache.
- */
- public static class RegionKey {
- /** Column key of cache used as input for {@link ColumnDecisionTreeTrainer}. */
- @AffinityKeyMapped
- private final Object parentColKey;
-
- /** Feature index. */
- private final int featureIdx;
-
- /** Region index. */
- private final int regBlockIdx;
-
- /** Training UUID. */
- private final UUID trainingUUID;
-
- /**
- * Construct a RegionKey from feature index, index of block, key of column in input cache and UUID of training.
- *
- * @param featureIdx Feature index.
- * @param regBlockIdx Index of block.
- * @param parentColKey Key of column in input cache.
- * @param trainingUUID UUID of training.
- */
- public RegionKey(int featureIdx, int regBlockIdx, Object parentColKey, UUID trainingUUID) {
- this.featureIdx = featureIdx;
- this.regBlockIdx = regBlockIdx;
- this.trainingUUID = trainingUUID;
- this.parentColKey = parentColKey;
- }
-
- /**
- * Feature index.
- *
- * @return Feature index.
- */
- public int featureIdx() {
- return featureIdx;
- }
-
- /**
- * Region block index.
- *
- * @return Region block index.
- */
- public int regionBlockIndex() {
- return regBlockIdx;
- }
-
- /**
- * UUID of training.
- *
- * @return UUID of training.
- */
- public UUID trainingUUID() {
- return trainingUUID;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- RegionKey key = (RegionKey)o;
-
- if (featureIdx != key.featureIdx)
- return false;
- if (regBlockIdx != key.regBlockIdx)
- return false;
- return trainingUUID != null ? trainingUUID.equals(key.trainingUUID) : key.trainingUUID == null;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = trainingUUID != null ? trainingUUID.hashCode() : 0;
- res = 31 * res + featureIdx;
- res = 31 * res + regBlockIdx;
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "RegionKey [" +
- "parentColKey=" + parentColKey +
- ", featureIdx=" + featureIdx +
- ", regBlockIdx=" + regBlockIdx +
- ", trainingUUID=" + trainingUUID +
- ']';
- }
- }
-
- /**
- * Affinity service for region projections cache.
- *
- * @return Affinity service for region projections cache.
- */
- public static Affinity<RegionKey> affinity() {
- return Ignition.localIgnite().affinity(CACHE_NAME);
- }
-
- /**
- * Get or create region projections cache.
- *
- * @param ignite Ignite instance.
- * @return Region projections cache.
- */
- public static IgniteCache<RegionKey, List<RegionProjection>> getOrCreate(Ignite ignite) {
- CacheConfiguration<RegionKey, List<RegionProjection>> cfg = new CacheConfiguration<>();
-
- // Write to primary.
- cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
- // Atomic transactions only.
- cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
- // No eviction.
- cfg.setEvictionPolicy(null);
-
- // No copying of values.
- cfg.setCopyOnRead(false);
-
- // Cache is partitioned.
- cfg.setCacheMode(CacheMode.PARTITIONED);
-
- cfg.setBackups(0);
-
- cfg.setOnheapCacheEnabled(true);
-
- cfg.setName(CACHE_NAME);
-
- return ignite.getOrCreateCache(cfg);
- }
-
- /**
- * Get region projections in the form of map (regionIndex -> regionProjections).
- *
- * @param featureIdx Feature index.
- * @param maxDepth Max depth of decision tree.
- * @param regionIndexes Indexes of regions for which we want get projections.
- * @param blockSize Size of regions block.
- * @param affinity Affinity function.
- * @param trainingUUID UUID of training.
- * @param ignite Ignite instance.
- * @return Region projections in the form of map (regionIndex -> regionProjections).
- */
- public static Map<Integer, RegionProjection> projectionsOfRegions(int featureIdx, int maxDepth,
- IntStream regionIndexes, int blockSize, IgniteFunction<Integer, Object> affinity, UUID trainingUUID,
- Ignite ignite) {
- HashMap<Integer, RegionProjection> regsForSearch = new HashMap<>();
- IgniteCache<RegionKey, List<RegionProjection>> cache = getOrCreate(ignite);
-
- PrimitiveIterator.OfInt itr = regionIndexes.iterator();
-
- int curBlockIdx = -1;
- List<RegionProjection> block = null;
-
- Object affinityKey = affinity.apply(featureIdx);
-
- while (itr.hasNext()) {
- int i = itr.nextInt();
-
- int blockIdx = i / blockSize;
-
- if (blockIdx != curBlockIdx) {
- block = cache.localPeek(key(featureIdx, blockIdx, affinityKey, trainingUUID));
- curBlockIdx = blockIdx;
- }
-
- if (block == null)
- throw new IllegalStateException("Unexpected null block at index " + i);
-
- RegionProjection reg = block.get(i % blockSize);
-
- if (reg.depth() < maxDepth)
- regsForSearch.put(i, reg);
- }
-
- return regsForSearch;
- }
-
- /**
- * Returns projections of regions on given feature filtered by maximal depth in the form of (region index -> region
- * projection).
- *
- * @param featureIdx Feature index.
- * @param maxDepth Maximal depth of the tree.
- * @param regsCnt Count of regions.
- * @param blockSize Size of regions blocks.
- * @param affinity Affinity function.
- * @param trainingUUID UUID of training.
- * @param ignite Ignite instance.
- * @return Projections of regions on given feature filtered by maximal depth in the form of (region index -> region
- * projection).
- */
- public static Map<Integer, RegionProjection> projectionsOfFeature(int featureIdx, int maxDepth, int regsCnt,
- int blockSize, IgniteFunction<Integer, Object> affinity, UUID trainingUUID, Ignite ignite) {
- return projectionsOfRegions(featureIdx, maxDepth, IntStream.range(0, regsCnt), blockSize, affinity, trainingUUID, ignite);
- }
-
- /**
- * Construct key for projections cache.
- *
- * @param featureIdx Feature index.
- * @param regBlockIdx Region block index.
- * @param parentColKey Column key of cache used as input for {@link ColumnDecisionTreeTrainer}.
- * @param uuid UUID of training.
- * @return Key for projections cache.
- */
- public static RegionKey key(int featureIdx, int regBlockIdx, Object parentColKey, UUID uuid) {
- return new RegionKey(featureIdx, regBlockIdx, parentColKey, uuid);
- }
-
- /**
- * Clear data from projections cache related to given training.
- *
- * @param featuresCnt Features count.
- * @param regs Regions count.
- * @param aff Affinity function.
- * @param uuid UUID of training.
- * @param ignite Ignite instance.
- */
- public static void clear(int featuresCnt, int regs, IgniteBiFunction<Integer, Ignite, Object> aff, UUID uuid,
- Ignite ignite) {
- Set<RegionKey> toRmv = IntStream.range(0, featuresCnt).boxed().
- flatMap(fIdx -> IntStream.range(0, regs).boxed().map(reg -> new IgniteBiTuple<>(fIdx, reg))).
- map(t -> key(t.get1(), t.get2(), aff.apply(t.get1(), ignite), uuid)).
- collect(Collectors.toSet());
-
- getOrCreate(ignite).removeAll(toRmv);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/SplitCache.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/SplitCache.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/SplitCache.java
deleted file mode 100644
index ecbc861..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/SplitCache.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Set;
-import java.util.UUID;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-
-/**
- * Class for working with cache used for storing of best splits during training with {@link ColumnDecisionTreeTrainer}.
- */
-public class SplitCache {
- /** Name of splits cache. */
- public static final String CACHE_NAME = "COLUMN_DECISION_TREE_TRAINER_SPLIT_CACHE_NAME";
-
- /**
- * Class used for keys in the splits cache.
- */
- public static class SplitKey {
- /** UUID of current training. */
- private final UUID trainingUUID;
-
- /** Affinity key of input data. */
- @AffinityKeyMapped
- private final Object parentColKey;
-
- /** Index of feature by which the split is made. */
- private final int featureIdx;
-
- /**
- * Construct SplitKey.
- *
- * @param trainingUUID UUID of the training.
- * @param parentColKey Affinity key used to ensure that cache entry for given feature will be on the same node
- * as column with that feature in input.
- * @param featureIdx Feature index.
- */
- public SplitKey(UUID trainingUUID, Object parentColKey, int featureIdx) {
- this.trainingUUID = trainingUUID;
- this.featureIdx = featureIdx;
- this.parentColKey = parentColKey;
- }
-
- /** Get UUID of current training. */
- public UUID trainingUUID() {
- return trainingUUID;
- }
-
- /**
- * Get feature index.
- *
- * @return Feature index.
- */
- public int featureIdx() {
- return featureIdx;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- SplitKey splitKey = (SplitKey)o;
-
- if (featureIdx != splitKey.featureIdx)
- return false;
- return trainingUUID != null ? trainingUUID.equals(splitKey.trainingUUID) : splitKey.trainingUUID == null;
-
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = trainingUUID != null ? trainingUUID.hashCode() : 0;
- res = 31 * res + featureIdx;
- return res;
- }
- }
-
- /**
- * Construct the key for splits cache.
- *
- * @param featureIdx Feature index.
- * @param parentColKey Affinity key used to ensure that cache entry for given feature will be on the same node as
- * column with that feature in input.
- * @param uuid UUID of current training.
- * @return Key for splits cache.
- */
- public static SplitKey key(int featureIdx, Object parentColKey, UUID uuid) {
- return new SplitKey(uuid, parentColKey, featureIdx);
- }
-
- /**
- * Get or create splits cache.
- *
- * @param ignite Ignite instance.
- * @return Splits cache.
- */
- public static IgniteCache<SplitKey, IgniteBiTuple<Integer, Double>> getOrCreate(Ignite ignite) {
- CacheConfiguration<SplitKey, IgniteBiTuple<Integer, Double>> cfg = new CacheConfiguration<>();
-
- // Write to primary.
- cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
- // Atomic transactions only.
- cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
- // No eviction.
- cfg.setEvictionPolicy(null);
-
- // No copying of values.
- cfg.setCopyOnRead(false);
-
- // Cache is partitioned.
- cfg.setCacheMode(CacheMode.PARTITIONED);
-
- cfg.setBackups(0);
-
- cfg.setOnheapCacheEnabled(true);
-
- cfg.setName(CACHE_NAME);
-
- return ignite.getOrCreateCache(cfg);
- }
-
- /**
- * Affinity function used in splits cache.
- *
- * @return Affinity function used in splits cache.
- */
- public static Affinity<SplitKey> affinity() {
- return Ignition.localIgnite().affinity(CACHE_NAME);
- }
-
- /**
- * Returns local entries for keys corresponding to {@code featureIndexes}.
- *
- * @param featureIndexes Index of features.
- * @param affinity Affinity function.
- * @param trainingUUID UUID of training.
- * @return local entries for keys corresponding to {@code featureIndexes}.
- */
- public static Iterable<Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>>> localEntries(
- Set<Integer> featureIndexes,
- IgniteBiFunction<Integer, Ignite, Object> affinity,
- UUID trainingUUID) {
- Ignite ignite = Ignition.localIgnite();
- Set<SplitKey> keys = featureIndexes.stream().map(fIdx -> new SplitKey(trainingUUID, affinity.apply(fIdx, ignite), fIdx)).collect(Collectors.toSet());
-
- Collection<SplitKey> locKeys = affinity().mapKeysToNodes(keys).getOrDefault(ignite.cluster().localNode(), Collections.emptyList());
-
- return () -> {
- Function<SplitKey, Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>>> f = k -> (new CacheEntryImpl<>(k, getOrCreate(ignite).localPeek(k)));
- return locKeys.stream().map(f).iterator();
- };
- }
-
- /**
- * Clears data related to current training from splits cache related to given training.
- *
- * @param featuresCnt Count of features.
- * @param affinity Affinity function.
- * @param uuid UUID of the given training.
- * @param ignite Ignite instance.
- */
- public static void clear(int featuresCnt, IgniteBiFunction<Integer, Ignite, Object> affinity, UUID uuid,
- Ignite ignite) {
- Set<SplitKey> toRmv = IntStream.range(0, featuresCnt).boxed().map(fIdx -> new SplitKey(uuid, affinity.apply(fIdx, ignite), fIdx)).collect(Collectors.toSet());
-
- getOrCreate(ignite).removeAll(toRmv);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/package-info.java
deleted file mode 100644
index 0a488ab..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains cache configurations for columnbased decision tree trainer with some related logic.
- */
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/ContinuousSplitCalculators.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/ContinuousSplitCalculators.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/ContinuousSplitCalculators.java
deleted file mode 100644
index 9fd4c66..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/ContinuousSplitCalculators.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.functions.IgniteCurriedBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainerInput;
-
-/** Continuous Split Calculators. */
-public class ContinuousSplitCalculators {
- /** Variance split calculator. */
- public static IgniteFunction<ColumnDecisionTreeTrainerInput, VarianceSplitCalculator> VARIANCE = input ->
- new VarianceSplitCalculator();
-
- /** Gini split calculator. */
- public static IgniteCurriedBiFunction<Ignite, ColumnDecisionTreeTrainerInput, GiniSplitCalculator> GINI = ignite ->
- input -> new GiniSplitCalculator(input.labels(ignite));
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/GiniSplitCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/GiniSplitCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/GiniSplitCalculator.java
deleted file mode 100644
index 259c84c..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/GiniSplitCalculator.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs;
-
-import it.unimi.dsi.fastutil.doubles.Double2IntArrayMap;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.PrimitiveIterator;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.ContinuousSplitInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-
-/**
- * Calculator for Gini impurity.
- */
-public class GiniSplitCalculator implements ContinuousSplitCalculator<GiniSplitCalculator.GiniData> {
- /** Mapping assigning index to each member value */
- private final Map<Double, Integer> mapping = new Double2IntArrayMap();
-
- /**
- * Create Gini split calculator from labels.
- *
- * @param labels Labels.
- */
- public GiniSplitCalculator(double[] labels) {
- int i = 0;
-
- for (double label : labels) {
- if (!mapping.containsKey(label)) {
- mapping.put(label, i);
- i++;
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public GiniData calculateRegionInfo(DoubleStream s, int l) {
- PrimitiveIterator.OfDouble itr = s.iterator();
-
- Map<Double, Integer> m = new HashMap<>();
-
- int size = 0;
-
- while (itr.hasNext()) {
- size++;
- m.compute(itr.next(), (a, i) -> i != null ? i + 1 : 1);
- }
-
- double c2 = m.values().stream().mapToDouble(v -> v * v).sum();
-
- int[] cnts = new int[mapping.size()];
-
- m.forEach((key, value) -> cnts[mapping.get(key)] = value);
-
- return new GiniData(size != 0 ? 1 - c2 / (size * size) : 0.0, size, cnts, c2);
- }
-
- /** {@inheritDoc} */
- @Override public SplitInfo<GiniData> splitRegion(Integer[] s, double[] values, double[] labels, int regionIdx,
- GiniData d) {
- int size = d.getSize();
-
- double lg = 0.0;
- double rg = d.impurity();
-
- double lc2 = 0.0;
- double rc2 = d.c2;
- int lSize = 0;
-
- double minImpurity = d.impurity() * size;
- double curThreshold;
- double curImpurity;
- double threshold = Double.NEGATIVE_INFINITY;
-
- int i = 0;
- int nextIdx = s[0];
- i++;
- double[] lrImps = new double[] {0.0, d.impurity(), lc2, rc2};
-
- int[] lMapCur = new int[d.counts().length];
- int[] rMapCur = new int[d.counts().length];
-
- System.arraycopy(d.counts(), 0, rMapCur, 0, d.counts().length);
-
- int[] lMap = new int[d.counts().length];
- int[] rMap = new int[d.counts().length];
-
- System.arraycopy(d.counts(), 0, rMap, 0, d.counts().length);
-
- do {
- // Process all values equal to prev.
- while (i < s.length) {
- moveLeft(labels[nextIdx], i, size - i, lMapCur, rMapCur, lrImps);
- curImpurity = (i * lrImps[0] + (size - i) * lrImps[1]);
- curThreshold = values[nextIdx];
-
- if (values[nextIdx] != values[(nextIdx = s[i++])]) {
- if (curImpurity < minImpurity) {
- lSize = i - 1;
-
- lg = lrImps[0];
- rg = lrImps[1];
-
- lc2 = lrImps[2];
- rc2 = lrImps[3];
-
- System.arraycopy(lMapCur, 0, lMap, 0, lMapCur.length);
- System.arraycopy(rMapCur, 0, rMap, 0, rMapCur.length);
-
- minImpurity = curImpurity;
- threshold = curThreshold;
- }
-
- break;
- }
- }
- }
- while (i < s.length - 1);
-
- if (lSize == size || lSize == 0)
- return null;
-
- GiniData lData = new GiniData(lg, lSize, lMap, lc2);
- int rSize = size - lSize;
- GiniData rData = new GiniData(rg, rSize, rMap, rc2);
-
- return new ContinuousSplitInfo<>(regionIdx, threshold, lData, rData);
- }
-
- /**
- * Add point to the left interval and remove it from the right interval and calculate necessary statistics on
- * intervals with new bounds.
- */
- private void moveLeft(double x, int lSize, int rSize, int[] lMap, int[] rMap, double[] data) {
- double lc2 = data[2];
- double rc2 = data[3];
-
- Integer idx = mapping.get(x);
-
- int cxl = lMap[idx];
- int cxr = rMap[idx];
-
- lc2 += 2 * cxl + 1;
- rc2 -= 2 * cxr - 1;
-
- lMap[idx] += 1;
- rMap[idx] -= 1;
-
- data[0] = 1 - lc2 / (lSize * lSize);
- data[1] = 1 - rc2 / (rSize * rSize);
-
- data[2] = lc2;
- data[3] = rc2;
- }
-
- /**
- * Data used for gini impurity calculations.
- */
- public static class GiniData extends ContinuousRegionInfo {
- /** Sum of squares of counts of each label. */
- private double c2;
-
- /** Counts of each label. On i-th position there is count of label which is mapped to index i. */
- private int[] m;
-
- /**
- * Create Gini data.
- *
- * @param impurity Impurity (i.e. Gini impurity).
- * @param size Count of samples.
- * @param m Counts of each label.
- * @param c2 Sum of squares of counts of each label.
- */
- public GiniData(double impurity, int size, int[] m, double c2) {
- super(impurity, size);
- this.m = m;
- this.c2 = c2;
- }
-
- /**
- * No-op constructor for serialization/deserialization..
- */
- public GiniData() {
- // No-op.
- }
-
- /** Get counts of each label. */
- public int[] counts() {
- return m;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- super.writeExternal(out);
- out.writeDouble(c2);
- out.writeInt(m.length);
- for (int i : m)
- out.writeInt(i);
-
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- super.readExternal(in);
-
- c2 = in.readDouble();
- int size = in.readInt();
- m = new int[size];
-
- for (int i = 0; i < size; i++)
- m[i] = in.readInt();
- }
- }
-}
[14/14] ignite git commit: Merge branches 'ignite-8201' and 'master'
of https://git-wip-us.apache.org/repos/asf/ignite into ignite-8201
Posted by ak...@apache.org.
Merge branches 'ignite-8201' and 'master' of https://git-wip-us.apache.org/repos/asf/ignite into ignite-8201
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/df938d56
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/df938d56
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/df938d56
Branch: refs/heads/ignite-8201
Commit: df938d562e9fb59938c1628be2163a5c273d21d4
Parents: a808932 f4de6df
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Wed Apr 11 14:00:06 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Apr 11 14:00:06 2018 +0700
----------------------------------------------------------------------
...ecisionTreeClassificationTrainerExample.java | 147 +++++
.../DecisionTreeRegressionTrainerExample.java | 124 ++++
.../ignite/examples/ml/tree/package-info.java | 22 +
.../examples/ml/trees/DecisionTreesExample.java | 354 ------------
.../ignite/examples/ml/trees/package-info.java | 22 -
.../apache/ignite/IgniteSystemProperties.java | 7 +
.../failure/StopNodeOrHaltFailureHandler.java | 2 +-
.../ignite/internal/GridKernalContext.java | 8 +
.../ignite/internal/GridKernalContextImpl.java | 10 +
.../apache/ignite/internal/IgniteKernal.java | 16 +-
.../org/apache/ignite/internal/IgnitionEx.java | 15 +-
.../GridClientConnectionManagerAdapter.java | 6 +
.../impl/GridTcpRouterNioListenerAdapter.java | 6 +
.../discovery/GridDiscoveryManager.java | 18 +-
.../pagemem/impl/PageMemoryNoStoreImpl.java | 17 +-
.../GridCachePartitionExchangeManager.java | 15 +-
.../cache/GridCacheSharedTtlCleanupManager.java | 44 +-
.../GridCacheDatabaseSharedManager.java | 60 +-
.../persistence/pagemem/PageMemoryImpl.java | 17 +-
.../wal/FileWriteAheadLogManager.java | 157 +++--
.../wal/FsyncModeFileWriteAheadLogManager.java | 34 +-
.../wal/reader/StandaloneGridKernalContext.java | 6 +
.../timeout/GridTimeoutProcessor.java | 105 ++--
.../ignite/internal/util/IgniteUtils.java | 7 +-
.../ignite/internal/util/StripedExecutor.java | 69 ++-
.../ignite/internal/util/nio/GridNioServer.java | 43 +-
.../util/nio/GridNioServerListener.java | 6 +
.../util/nio/GridNioServerListenerAdapter.java | 6 +
.../util/nio/ssl/BlockingSslHandler.java | 10 +-
.../worker/WorkersControlMXBeanImpl.java | 62 ++
.../ignite/internal/worker/WorkersRegistry.java | 80 +++
.../ignite/internal/worker/package-info.java | 22 +
.../ignite/mxbean/WorkersControlMXBean.java | 49 ++
.../communication/tcp/TcpCommunicationSpi.java | 41 +-
.../ignite/spi/discovery/tcp/ServerImpl.java | 51 +-
.../failure/AbstractFailureHandlerTest.java | 74 +++
.../ignite/failure/IoomFailureHandlerTest.java | 144 +++++
.../cache/CacheGroupsMetricsRebalanceTest.java | 31 +-
.../persistence/pagemem/PageMemoryImplTest.java | 9 +
.../internal/util/StripedExecutorTest.java | 2 +-
.../ignite/testframework/GridTestUtils.java | 2 +-
.../ignite/testsuites/IgniteBasicTestSuite.java | 2 +
.../main/java/org/apache/ignite/ml/Trainer.java | 3 -
.../org/apache/ignite/ml/tree/DecisionTree.java | 252 ++++++++
.../tree/DecisionTreeClassificationTrainer.java | 93 +++
.../ml/tree/DecisionTreeConditionalNode.java | 78 +++
.../ignite/ml/tree/DecisionTreeLeafNode.java | 48 ++
.../apache/ignite/ml/tree/DecisionTreeNode.java | 26 +
.../ml/tree/DecisionTreeRegressionTrainer.java | 60 ++
.../org/apache/ignite/ml/tree/TreeFilter.java | 38 ++
.../ignite/ml/tree/data/DecisionTreeData.java | 128 +++++
.../ml/tree/data/DecisionTreeDataBuilder.java | 73 +++
.../ignite/ml/tree/data/package-info.java | 22 +
.../ml/tree/impurity/ImpurityMeasure.java | 55 ++
.../impurity/ImpurityMeasureCalculator.java | 38 ++
.../tree/impurity/gini/GiniImpurityMeasure.java | 115 ++++
.../gini/GiniImpurityMeasureCalculator.java | 110 ++++
.../ml/tree/impurity/gini/package-info.java | 22 +
.../tree/impurity/mse/MSEImpurityMeasure.java | 133 +++++
.../mse/MSEImpurityMeasureCalculator.java | 80 +++
.../ml/tree/impurity/mse/package-info.java | 22 +
.../ignite/ml/tree/impurity/package-info.java | 22 +
.../util/SimpleStepFunctionCompressor.java | 149 +++++
.../ml/tree/impurity/util/StepFunction.java | 162 ++++++
.../impurity/util/StepFunctionCompressor.java | 55 ++
.../ml/tree/impurity/util/package-info.java | 22 +
.../ml/tree/leaf/DecisionTreeLeafBuilder.java | 38 ++
.../tree/leaf/MeanDecisionTreeLeafBuilder.java | 73 +++
.../leaf/MostCommonDecisionTreeLeafBuilder.java | 86 +++
.../ignite/ml/tree/leaf/package-info.java | 22 +
.../org/apache/ignite/ml/tree/package-info.java | 22 +
.../ignite/ml/trees/CategoricalRegionInfo.java | 72 ---
.../ignite/ml/trees/CategoricalSplitInfo.java | 68 ---
.../ignite/ml/trees/ContinuousRegionInfo.java | 74 ---
.../ml/trees/ContinuousSplitCalculator.java | 51 --
.../org/apache/ignite/ml/trees/RegionInfo.java | 62 --
.../ml/trees/models/DecisionTreeModel.java | 44 --
.../ignite/ml/trees/models/package-info.java | 22 -
.../ml/trees/nodes/CategoricalSplitNode.java | 50 --
.../ml/trees/nodes/ContinuousSplitNode.java | 56 --
.../ignite/ml/trees/nodes/DecisionTreeNode.java | 33 --
.../org/apache/ignite/ml/trees/nodes/Leaf.java | 49 --
.../apache/ignite/ml/trees/nodes/SplitNode.java | 100 ----
.../ignite/ml/trees/nodes/package-info.java | 22 -
.../apache/ignite/ml/trees/package-info.java | 22 -
.../ml/trees/trainers/columnbased/BiIndex.java | 113 ----
...exedCacheColumnDecisionTreeTrainerInput.java | 57 --
.../CacheColumnDecisionTreeTrainerInput.java | 141 -----
.../columnbased/ColumnDecisionTreeTrainer.java | 568 -------------------
.../ColumnDecisionTreeTrainerInput.java | 55 --
.../MatrixColumnDecisionTreeTrainerInput.java | 83 ---
.../trainers/columnbased/RegionProjection.java | 109 ----
.../trainers/columnbased/TrainingContext.java | 166 ------
.../columnbased/caches/ContextCache.java | 68 ---
.../columnbased/caches/FeaturesCache.java | 151 -----
.../columnbased/caches/ProjectionsCache.java | 286 ----------
.../trainers/columnbased/caches/SplitCache.java | 206 -------
.../columnbased/caches/package-info.java | 22 -
.../ContinuousSplitCalculators.java | 34 --
.../contsplitcalcs/GiniSplitCalculator.java | 234 --------
.../contsplitcalcs/VarianceSplitCalculator.java | 179 ------
.../contsplitcalcs/package-info.java | 22 -
.../trainers/columnbased/package-info.java | 22 -
.../columnbased/regcalcs/RegionCalculators.java | 85 ---
.../columnbased/regcalcs/package-info.java | 22 -
.../vectors/CategoricalFeatureProcessor.java | 212 -------
.../vectors/ContinuousFeatureProcessor.java | 111 ----
.../vectors/ContinuousSplitInfo.java | 71 ---
.../columnbased/vectors/FeatureProcessor.java | 82 ---
.../vectors/FeatureVectorProcessorUtils.java | 57 --
.../columnbased/vectors/SampleInfo.java | 80 ---
.../trainers/columnbased/vectors/SplitInfo.java | 106 ----
.../columnbased/vectors/package-info.java | 22 -
.../org/apache/ignite/ml/IgniteMLTestSuite.java | 4 +-
.../ml/nn/performance/MnistMLPTestUtil.java | 9 +-
...reeClassificationTrainerIntegrationTest.java | 100 ++++
.../DecisionTreeClassificationTrainerTest.java | 91 +++
...ionTreeRegressionTrainerIntegrationTest.java | 100 ++++
.../tree/DecisionTreeRegressionTrainerTest.java | 91 +++
.../ignite/ml/tree/DecisionTreeTestSuite.java | 48 ++
.../ml/tree/data/DecisionTreeDataTest.java | 59 ++
.../gini/GiniImpurityMeasureCalculatorTest.java | 103 ++++
.../impurity/gini/GiniImpurityMeasureTest.java | 131 +++++
.../mse/MSEImpurityMeasureCalculatorTest.java | 59 ++
.../impurity/mse/MSEImpurityMeasureTest.java | 109 ++++
.../util/SimpleStepFunctionCompressorTest.java | 75 +++
.../ml/tree/impurity/util/StepFunctionTest.java | 71 +++
.../tree/impurity/util/TestImpurityMeasure.java | 88 +++
.../DecisionTreeMNISTIntegrationTest.java | 105 ++++
.../tree/performance/DecisionTreeMNISTTest.java | 74 +++
.../ignite/ml/trees/BaseDecisionTreeTest.java | 70 ---
.../ml/trees/ColumnDecisionTreeTrainerTest.java | 191 -------
.../ignite/ml/trees/DecisionTreesTestSuite.java | 33 --
.../ml/trees/GiniSplitCalculatorTest.java | 141 -----
.../ignite/ml/trees/SplitDataGenerator.java | 390 -------------
.../ml/trees/VarianceSplitCalculatorTest.java | 84 ---
.../ColumnDecisionTreeTrainerBenchmark.java | 456 ---------------
modules/web-console/backend/routes/demo.js | 2 +
.../demo/service/DemoCachesLoadService.java | 22 +-
.../IgniteColumnDecisionTreeGiniBenchmark.java | 70 ---
...niteColumnDecisionTreeVarianceBenchmark.java | 71 ---
.../yardstick/ml/trees/SplitDataGenerator.java | 426 --------------
.../ignite/yardstick/ml/trees/package-info.java | 22 -
143 files changed, 4726 insertions(+), 6627 deletions(-)
----------------------------------------------------------------------
[02/14] ignite git commit: IGNITE-8059: Integrate decision tree with
partition based dataset.
Posted by ak...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureTest.java
new file mode 100644
index 0000000..3d11d9d
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.mse;
+
+import java.util.Random;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+
+/**
+ * Tests for {@link MSEImpurityMeasure}.
+ */
+public class MSEImpurityMeasureTest {
+ /** */
+ @Test
+ public void testImpurityOnEmptyData() {
+ MSEImpurityMeasure impurity = new MSEImpurityMeasure(0, 0, 0, 0, 0, 0);
+
+ assertEquals(0.0, impurity.impurity(), 1e-10);
+ }
+
+ /** */
+ @Test
+ public void testImpurityLeftPart() {
+ // Test on left part [1, 2, 2, 1, 1, 1].
+ MSEImpurityMeasure impurity = new MSEImpurityMeasure(8, 12, 6, 0, 0, 0);
+
+ assertEquals(1.333, impurity.impurity(), 1e-3);
+ }
+
+ /** */
+ @Test
+ public void testImpurityRightPart() {
+ // Test on right part [1, 2, 2, 1, 1, 1].
+ MSEImpurityMeasure impurity = new MSEImpurityMeasure(0, 0, 0, 8, 12, 6);
+
+ assertEquals(1.333, impurity.impurity(), 1e-3);
+ }
+
+ /** */
+ @Test
+ public void testImpurityLeftAndRightPart() {
+ // Test on left part [1, 2, 2] and right part [1, 1, 1].
+ MSEImpurityMeasure impurity = new MSEImpurityMeasure(5, 9, 3, 3, 3, 3);
+
+ assertEquals(0.666, impurity.impurity(), 1e-3);
+ }
+
+ /** */
+ @Test
+ public void testAdd() {
+ Random rnd = new Random(0);
+
+ MSEImpurityMeasure a = new MSEImpurityMeasure(
+ rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt(), rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt()
+ );
+
+ MSEImpurityMeasure b = new MSEImpurityMeasure(
+ rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt(), rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt()
+ );
+
+ MSEImpurityMeasure c = a.add(b);
+
+ assertEquals(a.getLeftY() + b.getLeftY(), c.getLeftY(), 1e-10);
+ assertEquals(a.getLeftY2() + b.getLeftY2(), c.getLeftY2(), 1e-10);
+ assertEquals(a.getLeftCnt() + b.getLeftCnt(), c.getLeftCnt());
+ assertEquals(a.getRightY() + b.getRightY(), c.getRightY(), 1e-10);
+ assertEquals(a.getRightY2() + b.getRightY2(), c.getRightY2(), 1e-10);
+ assertEquals(a.getRightCnt() + b.getRightCnt(), c.getRightCnt());
+ }
+
+ /** */
+ @Test
+ public void testSubtract() {
+ Random rnd = new Random(0);
+
+ MSEImpurityMeasure a = new MSEImpurityMeasure(
+ rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt(), rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt()
+ );
+
+ MSEImpurityMeasure b = new MSEImpurityMeasure(
+ rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt(), rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt()
+ );
+
+ MSEImpurityMeasure c = a.subtract(b);
+
+ assertEquals(a.getLeftY() - b.getLeftY(), c.getLeftY(), 1e-10);
+ assertEquals(a.getLeftY2() - b.getLeftY2(), c.getLeftY2(), 1e-10);
+ assertEquals(a.getLeftCnt() - b.getLeftCnt(), c.getLeftCnt());
+ assertEquals(a.getRightY() - b.getRightY(), c.getRightY(), 1e-10);
+ assertEquals(a.getRightY2() - b.getRightY2(), c.getRightY2(), 1e-10);
+ assertEquals(a.getRightCnt() - b.getRightCnt(), c.getRightCnt());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressorTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressorTest.java
new file mode 100644
index 0000000..001404f
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressorTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link SimpleStepFunctionCompressor}.
+ */
+public class SimpleStepFunctionCompressorTest {
+ /** */
+ @Test
+ public void testCompressSmallFunction() {
+ StepFunction<TestImpurityMeasure> function = new StepFunction<>(
+ new double[]{1, 2, 3, 4},
+ TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3, 4)
+ );
+
+ SimpleStepFunctionCompressor<TestImpurityMeasure> compressor = new SimpleStepFunctionCompressor<>(5, 0, 0);
+
+ StepFunction<TestImpurityMeasure> resFunction = compressor.compress(function);
+
+ assertArrayEquals(new double[]{1, 2, 3, 4}, resFunction.getX(), 1e-10);
+ assertArrayEquals(TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3, 4), resFunction.getY());
+ }
+
+ /** */
+ @Test
+ public void testCompressIncreasingFunction() {
+ StepFunction<TestImpurityMeasure> function = new StepFunction<>(
+ new double[]{1, 2, 3, 4, 5},
+ TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3, 4, 5)
+ );
+
+ SimpleStepFunctionCompressor<TestImpurityMeasure> compressor = new SimpleStepFunctionCompressor<>(1, 0.4, 0);
+
+ StepFunction<TestImpurityMeasure> resFunction = compressor.compress(function);
+
+ assertArrayEquals(new double[]{1, 3, 5}, resFunction.getX(), 1e-10);
+ assertArrayEquals(TestImpurityMeasure.asTestImpurityMeasures(1, 3, 5), resFunction.getY());
+ }
+
+ /** */
+ @Test
+ public void testCompressDecreasingFunction() {
+ StepFunction<TestImpurityMeasure> function = new StepFunction<>(
+ new double[]{1, 2, 3, 4, 5},
+ TestImpurityMeasure.asTestImpurityMeasures(5, 4, 3, 2, 1)
+ );
+
+ SimpleStepFunctionCompressor<TestImpurityMeasure> compressor = new SimpleStepFunctionCompressor<>(1, 0, 0.4);
+
+ StepFunction<TestImpurityMeasure> resFunction = compressor.compress(function);
+
+ assertArrayEquals(new double[]{1, 3, 5}, resFunction.getX(), 1e-10);
+ assertArrayEquals(TestImpurityMeasure.asTestImpurityMeasures(5, 3, 1), resFunction.getY());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionTest.java
new file mode 100644
index 0000000..2a0279c
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link StepFunction}.
+ */
+public class StepFunctionTest {
+ /** */
+ @Test
+ public void testAddIncreasingFunctions() {
+ StepFunction<TestImpurityMeasure> a = new StepFunction<>(
+ new double[]{1, 3, 5},
+ TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3)
+ );
+
+ StepFunction<TestImpurityMeasure> b = new StepFunction<>(
+ new double[]{0, 2, 4},
+ TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3)
+ );
+
+ StepFunction<TestImpurityMeasure> c = a.add(b);
+
+ assertArrayEquals(new double[]{0, 1, 2, 3, 4, 5}, c.getX(), 1e-10);
+ assertArrayEquals(
+ TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3, 4, 5, 6),
+ c.getY()
+ );
+ }
+
+ /** */
+ @Test
+ public void testAddDecreasingFunctions() {
+ StepFunction<TestImpurityMeasure> a = new StepFunction<>(
+ new double[]{1, 3, 5},
+ TestImpurityMeasure.asTestImpurityMeasures(3, 2, 1)
+ );
+
+ StepFunction<TestImpurityMeasure> b = new StepFunction<>(
+ new double[]{0, 2, 4},
+ TestImpurityMeasure.asTestImpurityMeasures(3, 2, 1)
+ );
+
+ StepFunction<TestImpurityMeasure> c = a.add(b);
+
+ assertArrayEquals(new double[]{0, 1, 2, 3, 4, 5}, c.getX(), 1e-10);
+ assertArrayEquals(
+ TestImpurityMeasure.asTestImpurityMeasures(3, 6, 5, 4, 3, 2),
+ c.getY()
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/TestImpurityMeasure.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/TestImpurityMeasure.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/TestImpurityMeasure.java
new file mode 100644
index 0000000..c0d1911
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/TestImpurityMeasure.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import java.util.Objects;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Utils class used as impurity measure in tests.
+ */
+class TestImpurityMeasure implements ImpurityMeasure<TestImpurityMeasure> {
+ /** */
+ private static final long serialVersionUID = 2414020770162797847L;
+
+ /** Impurity. */
+ private final double impurity;
+
+ /**
+ * Constructs a new instance of test impurity measure.
+ *
+ * @param impurity Impurity.
+ */
+ private TestImpurityMeasure(double impurity) {
+ this.impurity = impurity;
+ }
+
+ /**
+ * Convert doubles to array of test impurity measures.
+ *
+ * @param impurity Impurity as array of doubles.
+ * @return Test impurity measure objects as array.
+ */
+ static TestImpurityMeasure[] asTestImpurityMeasures(double... impurity) {
+ TestImpurityMeasure[] res = new TestImpurityMeasure[impurity.length];
+
+ for (int i = 0; i < impurity.length; i++)
+ res[i] = new TestImpurityMeasure(impurity[i]);
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public double impurity() {
+ return impurity;
+ }
+
+ /** {@inheritDoc} */
+ @Override public TestImpurityMeasure add(TestImpurityMeasure measure) {
+ return new TestImpurityMeasure(impurity + measure.impurity);
+ }
+
+ /** {@inheritDoc} */
+ @Override public TestImpurityMeasure subtract(TestImpurityMeasure measure) {
+ return new TestImpurityMeasure(impurity - measure.impurity);
+ }
+
+ /** */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ TestImpurityMeasure measure = (TestImpurityMeasure)o;
+
+ return Double.compare(measure.impurity, impurity) == 0;
+ }
+
+ /** */
+ @Override public int hashCode() {
+
+ return Objects.hash(impurity);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
new file mode 100644
index 0000000..b259ec9
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.performance;
+
+import java.io.IOException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.nn.performance.MnistMLPTestUtil;
+import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
+import org.apache.ignite.ml.tree.DecisionTreeNode;
+import org.apache.ignite.ml.tree.impurity.util.SimpleStepFunctionCompressor;
+import org.apache.ignite.ml.util.MnistUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests {@link DecisionTreeClassificationTrainer} on the MNIST dataset that require to start the whole Ignite
+ * infrastructure. For manual run.
+ */
+public class DecisionTreeMNISTIntegrationTest extends GridCommonAbstractTest {
+ /** Number of nodes in grid */
+ private static final int NODE_COUNT = 3;
+
+ /** Ignite instance. */
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ for (int i = 1; i <= NODE_COUNT; i++)
+ startGrid(i);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() {
+ stopAllGrids();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void beforeTest() throws Exception {
+ /* Grid instance. */
+ ignite = grid(NODE_COUNT);
+ ignite.configuration().setPeerClassLoadingEnabled(true);
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+ }
+
+ /** Tests on the MNIST dataset. For manual run. */
+ public void testMNIST() throws IOException {
+ CacheConfiguration<Integer, MnistUtils.MnistLabeledImage> trainingSetCacheCfg = new CacheConfiguration<>();
+ trainingSetCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+ trainingSetCacheCfg.setName("MNIST_TRAINING_SET");
+
+ IgniteCache<Integer, MnistUtils.MnistLabeledImage> trainingSet = ignite.createCache(trainingSetCacheCfg);
+
+ int i = 0;
+ for (MnistUtils.MnistLabeledImage e : MnistMLPTestUtil.loadTrainingSet(60_000))
+ trainingSet.put(i++, e);
+
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(
+ 8,
+ 0,
+ new SimpleStepFunctionCompressor<>());
+
+ DecisionTreeNode mdl = trainer.fit(
+ new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+ (k, v) -> v.getPixels(),
+ (k, v) -> (double) v.getLabel()
+ );
+
+ int correctAnswers = 0;
+ int incorrectAnswers = 0;
+
+ for (MnistUtils.MnistLabeledImage e : MnistMLPTestUtil.loadTestSet(10_000)) {
+ double res = mdl.apply(e.getPixels());
+
+ if (res == e.getLabel())
+ correctAnswers++;
+ else
+ incorrectAnswers++;
+ }
+
+ double accuracy = 1.0 * correctAnswers / (correctAnswers + incorrectAnswers);
+
+ assertTrue(accuracy > 0.8);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
new file mode 100644
index 0000000..6dbd44c
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.performance;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.nn.performance.MnistMLPTestUtil;
+import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
+import org.apache.ignite.ml.tree.DecisionTreeNode;
+import org.apache.ignite.ml.tree.impurity.util.SimpleStepFunctionCompressor;
+import org.apache.ignite.ml.util.MnistUtils;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertTrue;
+
+/**
+ * Tests {@link DecisionTreeClassificationTrainer} on the MNIST dataset using locally stored data. For manual run.
+ */
+public class DecisionTreeMNISTTest {
+ /** Tests on the MNIST dataset. For manual run. */
+ @Test
+ public void testMNIST() throws IOException {
+ Map<Integer, MnistUtils.MnistLabeledImage> trainingSet = new HashMap<>();
+
+ int i = 0;
+ for (MnistUtils.MnistLabeledImage e : MnistMLPTestUtil.loadTrainingSet(60_000))
+ trainingSet.put(i++, e);
+
+
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(
+ 8,
+ 0,
+ new SimpleStepFunctionCompressor<>());
+
+ DecisionTreeNode mdl = trainer.fit(
+ new LocalDatasetBuilder<>(trainingSet, 10),
+ (k, v) -> v.getPixels(),
+ (k, v) -> (double) v.getLabel()
+ );
+
+ int correctAnswers = 0;
+ int incorrectAnswers = 0;
+
+ for (MnistUtils.MnistLabeledImage e : MnistMLPTestUtil.loadTestSet(10_000)) {
+ double res = mdl.apply(e.getPixels());
+
+ if (res == e.getLabel())
+ correctAnswers++;
+ else
+ incorrectAnswers++;
+ }
+
+ double accuracy = 1.0 * correctAnswers / (correctAnswers + incorrectAnswers);
+
+ assertTrue(accuracy > 0.8);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/trees/BaseDecisionTreeTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/BaseDecisionTreeTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/BaseDecisionTreeTest.java
deleted file mode 100644
index 65f0ae4..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/BaseDecisionTreeTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.Arrays;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledVectorDouble;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Base class for decision trees test.
- */
-public class BaseDecisionTreeTest extends GridCommonAbstractTest {
- /** Count of nodes. */
- private static final int NODE_COUNT = 4;
-
- /** Grid instance. */
- protected Ignite ignite;
-
- /**
- * Default constructor.
- */
- public BaseDecisionTreeTest() {
- super(false);
- }
-
- /**
- * {@inheritDoc}
- */
- @Override protected void beforeTest() throws Exception {
- ignite = grid(NODE_COUNT);
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- for (int i = 1; i <= NODE_COUNT; i++)
- startGrid(i);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- stopAllGrids();
- }
-
- /**
- * Convert double array to {@link LabeledVectorDouble}
- *
- * @param arr Array for conversion.
- * @return LabeledVectorDouble.
- */
- protected static LabeledVectorDouble<DenseLocalOnHeapVector> asLabeledVector(double arr[]) {
- return new LabeledVectorDouble<>(new DenseLocalOnHeapVector(Arrays.copyOf(arr, arr.length - 1)), arr[arr.length - 1]);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/trees/ColumnDecisionTreeTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/ColumnDecisionTreeTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/ColumnDecisionTreeTrainerTest.java
deleted file mode 100644
index b090f43..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/ColumnDecisionTreeTrainerTest.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Tracer;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledVectorDouble;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.MatrixColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-
-/** Tests behaviour of ColumnDecisionTreeTrainer. */
-public class ColumnDecisionTreeTrainerTest extends BaseDecisionTreeTest {
- /**
- * Test {@link ColumnDecisionTreeTrainerTest} for mixed (continuous and categorical) data with Gini impurity.
- */
- public void testCacheMixedGini() {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- int totalPts = 1 << 10;
- int featCnt = 2;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
- catsInfo.put(1, 3);
-
- Random rnd = new Random(12349L);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
- split(0, 1, new int[] {0, 2}).
- split(1, 0, -10.0);
-
- testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.GINI.apply(ignite), RegionCalculators.GINI, RegionCalculators.MEAN, rnd);
- }
-
- /**
- * Test {@link ColumnDecisionTreeTrainerTest} for mixed (continuous and categorical) data with Variance impurity.
- */
- public void testCacheMixed() {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- int totalPts = 1 << 10;
- int featCnt = 2;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
- catsInfo.put(1, 3);
-
- Random rnd = new Random(12349L);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
- split(0, 1, new int[] {0, 2}).
- split(1, 0, -10.0);
-
- testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, RegionCalculators.MEAN, rnd);
- }
-
- /**
- * Test {@link ColumnDecisionTreeTrainerTest} for continuous data with Variance impurity.
- */
- public void testCacheCont() {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- int totalPts = 1 << 10;
- int featCnt = 12;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
-
- Random rnd = new Random(12349L);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
- split(0, 0, -10.0).
- split(1, 0, 0.0).
- split(1, 1, 2.0).
- split(3, 7, 50.0);
-
- testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, RegionCalculators.MEAN, rnd);
- }
-
- /**
- * Test {@link ColumnDecisionTreeTrainerTest} for continuous data with Gini impurity.
- */
- public void testCacheContGini() {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- int totalPts = 1 << 10;
- int featCnt = 12;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
-
- Random rnd = new Random(12349L);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
- split(0, 0, -10.0).
- split(1, 0, 0.0).
- split(1, 1, 2.0).
- split(3, 7, 50.0);
-
- testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.GINI.apply(ignite), RegionCalculators.GINI, RegionCalculators.MEAN, rnd);
- }
-
- /**
- * Test {@link ColumnDecisionTreeTrainerTest} for categorical data with Variance impurity.
- */
- public void testCacheCat() {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- int totalPts = 1 << 10;
- int featCnt = 12;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
- catsInfo.put(5, 7);
-
- Random rnd = new Random(12349L);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
- split(0, 5, new int[] {0, 2, 5});
-
- testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, RegionCalculators.MEAN, rnd);
- }
-
- /** */
- private <D extends ContinuousRegionInfo> void testByGen(int totalPts, HashMap<Integer, Integer> catsInfo,
- SplitDataGenerator<DenseLocalOnHeapVector> gen,
- IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> calc,
- IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> catImpCalc,
- IgniteFunction<DoubleStream, Double> regCalc, Random rnd) {
-
- List<IgniteBiTuple<Integer, DenseLocalOnHeapVector>> lst = gen.
- points(totalPts, (i, rn) -> i).
- collect(Collectors.toList());
-
- int featCnt = gen.featuresCnt();
-
- Collections.shuffle(lst, rnd);
-
- SparseDistributedMatrix m = new SparseDistributedMatrix(totalPts, featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
- Map<Integer, List<LabeledVectorDouble>> byRegion = new HashMap<>();
-
- int i = 0;
- for (IgniteBiTuple<Integer, DenseLocalOnHeapVector> bt : lst) {
- byRegion.putIfAbsent(bt.get1(), new LinkedList<>());
- byRegion.get(bt.get1()).add(asLabeledVector(bt.get2().getStorage().data()));
- m.setRow(i, bt.get2().getStorage().data());
- i++;
- }
-
- ColumnDecisionTreeTrainer<D> trainer =
- new ColumnDecisionTreeTrainer<>(3, calc, catImpCalc, regCalc, ignite);
-
- DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, catsInfo));
-
- byRegion.keySet().forEach(k -> {
- LabeledVectorDouble sp = byRegion.get(k).get(0);
- Tracer.showAscii(sp.features());
- X.println("Actual and predicted vectors [act=" + sp.label() + " " + ", pred=" + mdl.apply(sp.features()) + "]");
- assert mdl.apply(sp.features()) == sp.doubleLabel();
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/trees/DecisionTreesTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/DecisionTreesTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/DecisionTreesTestSuite.java
deleted file mode 100644
index 3343503..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/DecisionTreesTestSuite.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-
-/**
- * Test suite for all tests located in org.apache.ignite.ml.trees package
- */
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
- ColumnDecisionTreeTrainerTest.class,
- GiniSplitCalculatorTest.class,
- VarianceSplitCalculatorTest.class
-})
-public class DecisionTreesTestSuite {
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/trees/GiniSplitCalculatorTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/GiniSplitCalculatorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/GiniSplitCalculatorTest.java
deleted file mode 100644
index c92b4f5..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/GiniSplitCalculatorTest.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.GiniSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-import org.junit.Test;
-
-/**
- * Test of {@link GiniSplitCalculator}.
- */
-public class GiniSplitCalculatorTest {
- /** Test calculation of region info consisting from one point. */
- @Test
- public void testCalculateRegionInfoSimple() {
- double labels[] = new double[] {0.0};
-
- assert new GiniSplitCalculator(labels).calculateRegionInfo(DoubleStream.of(labels), 0).impurity() == 0.0;
- }
-
- /** Test calculation of region info consisting from two distinct classes. */
- @Test
- public void testCalculateRegionInfoTwoClasses() {
- double labels[] = new double[] {0.0, 1.0};
-
- assert new GiniSplitCalculator(labels).calculateRegionInfo(DoubleStream.of(labels), 0).impurity() == 0.5;
- }
-
- /** Test calculation of region info consisting from three distinct classes. */
- @Test
- public void testCalculateRegionInfoThreeClasses() {
- double labels[] = new double[] {0.0, 1.0, 2.0};
-
- assert Math.abs(new GiniSplitCalculator(labels).calculateRegionInfo(DoubleStream.of(labels), 0).impurity() - 2.0 / 3) < 1E-5;
- }
-
- /** Test calculation of split of region consisting from one point. */
- @Test
- public void testSplitSimple() {
- double labels[] = new double[] {0.0};
- double values[] = new double[] {0.0};
- Integer[] samples = new Integer[] {0};
-
- int cnts[] = new int[] {1};
-
- GiniSplitCalculator.GiniData data = new GiniSplitCalculator.GiniData(0.0, 1, cnts, 1);
-
- assert new GiniSplitCalculator(labels).splitRegion(samples, values, labels, 0, data) == null;
- }
-
- /** Test calculation of split of region consisting from two points. */
- @Test
- public void testSplitTwoClassesTwoPoints() {
- double labels[] = new double[] {0.0, 1.0};
- double values[] = new double[] {0.0, 1.0};
- Integer[] samples = new Integer[] {0, 1};
-
- int cnts[] = new int[] {1, 1};
-
- GiniSplitCalculator.GiniData data = new GiniSplitCalculator.GiniData(0.5, 2, cnts, 1.0 * 1.0 + 1.0 * 1.0);
-
- SplitInfo<GiniSplitCalculator.GiniData> split = new GiniSplitCalculator(labels).splitRegion(samples, values, labels, 0, data);
-
- assert split.leftData().impurity() == 0;
- assert split.leftData().counts()[0] == 1;
- assert split.leftData().counts()[1] == 0;
- assert split.leftData().getSize() == 1;
-
- assert split.rightData().impurity() == 0;
- assert split.rightData().counts()[0] == 0;
- assert split.rightData().counts()[1] == 1;
- assert split.rightData().getSize() == 1;
- }
-
- /** Test calculation of split of region consisting from four distinct values. */
- @Test
- public void testSplitTwoClassesFourPoints() {
- double labels[] = new double[] {0.0, 0.0, 1.0, 1.0};
- double values[] = new double[] {0.0, 1.0, 2.0, 3.0};
-
- Integer[] samples = new Integer[] {0, 1, 2, 3};
-
- int[] cnts = new int[] {2, 2};
-
- GiniSplitCalculator.GiniData data = new GiniSplitCalculator.GiniData(0.5, 4, cnts, 2.0 * 2.0 + 2.0 * 2.0);
-
- SplitInfo<GiniSplitCalculator.GiniData> split = new GiniSplitCalculator(labels).splitRegion(samples, values, labels, 0, data);
-
- assert split.leftData().impurity() == 0;
- assert split.leftData().counts()[0] == 2;
- assert split.leftData().counts()[1] == 0;
- assert split.leftData().getSize() == 2;
-
- assert split.rightData().impurity() == 0;
- assert split.rightData().counts()[0] == 0;
- assert split.rightData().counts()[1] == 2;
- assert split.rightData().getSize() == 2;
- }
-
- /** Test calculation of split of region consisting from three distinct values. */
- @Test
- public void testSplitThreePoints() {
- double labels[] = new double[] {0.0, 1.0, 2.0};
- double values[] = new double[] {0.0, 1.0, 2.0};
- Integer[] samples = new Integer[] {0, 1, 2};
-
- int[] cnts = new int[] {1, 1, 1};
-
- GiniSplitCalculator.GiniData data = new GiniSplitCalculator.GiniData(2.0 / 3, 3, cnts, 1.0 * 1.0 + 1.0 * 1.0 + 1.0 * 1.0);
-
- SplitInfo<GiniSplitCalculator.GiniData> split = new GiniSplitCalculator(labels).splitRegion(samples, values, labels, 0, data);
-
- assert split.leftData().impurity() == 0.0;
- assert split.leftData().counts()[0] == 1;
- assert split.leftData().counts()[1] == 0;
- assert split.leftData().counts()[2] == 0;
- assert split.leftData().getSize() == 1;
-
- assert split.rightData().impurity() == 0.5;
- assert split.rightData().counts()[0] == 0;
- assert split.rightData().counts()[1] == 1;
- assert split.rightData().counts()[2] == 1;
- assert split.rightData().getSize() == 2;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/trees/SplitDataGenerator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/SplitDataGenerator.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/SplitDataGenerator.java
deleted file mode 100644
index 279e685..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/SplitDataGenerator.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ignite.ml.trees;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-import org.apache.ignite.ml.util.Utils;
-
-/**
- * Utility class for generating data which has binary tree split structure.
- *
- * @param <V>
- */
-public class SplitDataGenerator<V extends Vector> {
- /** */
- private static final double DELTA = 100.0;
-
- /** Map of the form of (is categorical -> list of region indexes). */
- private final Map<Boolean, List<Integer>> di;
-
- /** List of regions. */
- private final List<Region> regs;
-
- /** Data of bounds of regions. */
- private final Map<Integer, IgniteBiTuple<Double, Double>> boundsData;
-
- /** Random numbers generator. */
- private final Random rnd;
-
- /** Supplier of vectors. */
- private final Supplier<V> supplier;
-
- /** Features count. */
- private final int featCnt;
-
- /**
- * Create SplitDataGenerator.
- *
- * @param featCnt Features count.
- * @param catFeaturesInfo Information about categorical features in form of map (feature index -> categories
- * count).
- * @param supplier Supplier of vectors.
- * @param rnd Random numbers generator.
- */
- public SplitDataGenerator(int featCnt, Map<Integer, Integer> catFeaturesInfo, Supplier<V> supplier, Random rnd) {
- regs = new LinkedList<>();
- boundsData = new HashMap<>();
- this.rnd = rnd;
- this.supplier = supplier;
- this.featCnt = featCnt;
-
- // Divide indexes into indexes of categorical coordinates and indexes of continuous coordinates.
- di = IntStream.range(0, featCnt).
- boxed().
- collect(Collectors.partitioningBy(catFeaturesInfo::containsKey));
-
- // Categorical coordinates info.
- Map<Integer, CatCoordInfo> catCoords = new HashMap<>();
- di.get(true).forEach(i -> {
- BitSet bs = new BitSet();
- bs.set(0, catFeaturesInfo.get(i));
- catCoords.put(i, new CatCoordInfo(bs));
- });
-
- // Continuous coordinates info.
- Map<Integer, ContCoordInfo> contCoords = new HashMap<>();
- di.get(false).forEach(i -> {
- contCoords.put(i, new ContCoordInfo());
- boundsData.put(i, new IgniteBiTuple<>(-1.0, 1.0));
- });
-
- Region firstReg = new Region(catCoords, contCoords, 0);
- regs.add(firstReg);
- }
-
- /**
- * Categorical coordinate info.
- */
- private static class CatCoordInfo implements Serializable {
- /**
- * Defines categories which are included in this region
- */
- private final BitSet bs;
-
- /**
- * Construct CatCoordInfo.
- *
- * @param bs Bitset.
- */
- CatCoordInfo(BitSet bs) {
- this.bs = bs;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "CatCoordInfo [" +
- "bs=" + bs +
- ']';
- }
- }
-
- /**
- * Continuous coordinate info.
- */
- private static class ContCoordInfo implements Serializable {
- /**
- * Left (min) bound of region.
- */
- private double left;
-
- /**
- * Right (max) bound of region.
- */
- private double right;
-
- /**
- * Construct ContCoordInfo.
- */
- ContCoordInfo() {
- left = Double.NEGATIVE_INFINITY;
- right = Double.POSITIVE_INFINITY;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "ContCoordInfo [" +
- "left=" + left +
- ", right=" + right +
- ']';
- }
- }
-
- /**
- * Class representing information about region.
- */
- private static class Region implements Serializable {
- /**
- * Information about categorical coordinates restrictions of this region in form of
- * (coordinate index -> restriction)
- */
- private final Map<Integer, CatCoordInfo> catCoords;
-
- /**
- * Information about continuous coordinates restrictions of this region in form of
- * (coordinate index -> restriction)
- */
- private final Map<Integer, ContCoordInfo> contCoords;
-
- /**
- * Region should contain {@code 1/2^twoPow * totalPoints} points.
- */
- private int twoPow;
-
- /**
- * Construct region by information about restrictions on coordinates (features) values.
- *
- * @param catCoords Restrictions on categorical coordinates.
- * @param contCoords Restrictions on continuous coordinates
- * @param twoPow Region should contain {@code 1/2^twoPow * totalPoints} points.
- */
- Region(Map<Integer, CatCoordInfo> catCoords, Map<Integer, ContCoordInfo> contCoords, int twoPow) {
- this.catCoords = catCoords;
- this.contCoords = contCoords;
- this.twoPow = twoPow;
- }
-
- /** */
- int divideBy() {
- return 1 << twoPow;
- }
-
- /** */
- void incTwoPow() {
- twoPow++;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "Region [" +
- "catCoords=" + catCoords +
- ", contCoords=" + contCoords +
- ", twoPow=" + twoPow +
- ']';
- }
-
- /**
- * Generate continuous coordinate for this region.
- *
- * @param coordIdx Coordinate index.
- * @param boundsData Data with bounds
- * @param rnd Random numbers generator.
- * @return Categorical coordinate value.
- */
- double generateContCoord(int coordIdx, Map<Integer, IgniteBiTuple<Double, Double>> boundsData,
- Random rnd) {
- ContCoordInfo cci = contCoords.get(coordIdx);
- double left = cci.left;
- double right = cci.right;
-
- if (left == Double.NEGATIVE_INFINITY)
- left = boundsData.get(coordIdx).get1() - DELTA;
-
- if (right == Double.POSITIVE_INFINITY)
- right = boundsData.get(coordIdx).get2() + DELTA;
-
- double size = right - left;
-
- return left + rnd.nextDouble() * size;
- }
-
- /**
- * Generate categorical coordinate value for this region.
- *
- * @param coordIdx Coordinate index.
- * @param rnd Random numbers generator.
- * @return Categorical coordinate value.
- */
- double generateCatCoord(int coordIdx, Random rnd) {
- // Pick random bit.
- BitSet bs = catCoords.get(coordIdx).bs;
- int j = rnd.nextInt(bs.length());
-
- int i = 0;
- int bn = 0;
- int bnp = 0;
-
- while ((bn = bs.nextSetBit(bn)) != -1 && i <= j) {
- i++;
- bnp = bn;
- bn++;
- }
-
- return bnp;
- }
-
- /**
- * Generate points for this region.
- *
- * @param ptsCnt Count of points to generate.
- * @param val Label for all points in this region.
- * @param boundsData Data about bounds of continuous coordinates.
- * @param catCont Data about which categories can be in this region in the form (coordinate index -> list of
- * categories indexes).
- * @param s Vectors supplier.
- * @param rnd Random numbers generator.
- * @param <V> Type of vectors.
- * @return Stream of generated points for this region.
- */
- <V extends Vector> Stream<V> generatePoints(int ptsCnt, double val,
- Map<Integer, IgniteBiTuple<Double, Double>> boundsData, Map<Boolean, List<Integer>> catCont,
- Supplier<V> s,
- Random rnd) {
- return IntStream.range(0, ptsCnt / divideBy()).mapToObj(i -> {
- V v = s.get();
- int coordsCnt = v.size();
- catCont.get(false).forEach(ci -> v.setX(ci, generateContCoord(ci, boundsData, rnd)));
- catCont.get(true).forEach(ci -> v.setX(ci, generateCatCoord(ci, rnd)));
-
- v.setX(coordsCnt - 1, val);
- return v;
- });
- }
- }
-
- /**
- * Split region by continuous coordinate.using given threshold.
- *
- * @param regIdx Region index.
- * @param coordIdx Coordinate index.
- * @param threshold Threshold.
- * @return {@code this}.
- */
- public SplitDataGenerator<V> split(int regIdx, int coordIdx, double threshold) {
- Region regToSplit = regs.get(regIdx);
- ContCoordInfo cci = regToSplit.contCoords.get(coordIdx);
-
- double left = cci.left;
- double right = cci.right;
-
- if (threshold < left || threshold > right)
- throw new MathIllegalArgumentException("Threshold is out of region bounds.");
-
- regToSplit.incTwoPow();
-
- Region newReg = Utils.copy(regToSplit);
- newReg.contCoords.get(coordIdx).left = threshold;
-
- regs.add(regIdx + 1, newReg);
- cci.right = threshold;
-
- IgniteBiTuple<Double, Double> bounds = boundsData.get(coordIdx);
- double min = bounds.get1();
- double max = bounds.get2();
- boundsData.put(coordIdx, new IgniteBiTuple<>(Math.min(threshold, min), Math.max(max, threshold)));
-
- return this;
- }
-
- /**
- * Split region by categorical coordinate.
- *
- * @param regIdx Region index.
- * @param coordIdx Coordinate index.
- * @param cats Categories allowed for the left sub region.
- * @return {@code this}.
- */
- public SplitDataGenerator<V> split(int regIdx, int coordIdx, int[] cats) {
- BitSet subset = new BitSet();
- Arrays.stream(cats).forEach(subset::set);
- Region regToSplit = regs.get(regIdx);
- CatCoordInfo cci = regToSplit.catCoords.get(coordIdx);
-
- BitSet ssc = (BitSet)subset.clone();
- BitSet set = cci.bs;
- ssc.and(set);
- if (ssc.length() != subset.length())
- throw new MathIllegalArgumentException("Splitter set is not a subset of a parent subset.");
-
- ssc.xor(set);
- set.and(subset);
-
- regToSplit.incTwoPow();
- Region newReg = Utils.copy(regToSplit);
- newReg.catCoords.put(coordIdx, new CatCoordInfo(ssc));
-
- regs.add(regIdx + 1, newReg);
-
- return this;
- }
-
- /**
- * Get stream of points generated by this generator.
- *
- * @param ptsCnt Points count.
- */
- public Stream<IgniteBiTuple<Integer, V>> points(int ptsCnt, BiFunction<Double, Random, Double> f) {
- regs.forEach(System.out::println);
-
- return IntStream.range(0, regs.size()).
- boxed().
- map(i -> regs.get(i).generatePoints(ptsCnt, f.apply((double)i, rnd), boundsData, di, supplier, rnd).map(v -> new IgniteBiTuple<>(i, v))).flatMap(Function.identity());
- }
-
- /**
- * Count of regions.
- *
- * @return Count of regions.
- */
- public int regsCount() {
- return regs.size();
- }
-
- /**
- * Get features count.
- *
- * @return Features count.
- */
- public int featuresCnt() {
- return featCnt;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/trees/VarianceSplitCalculatorTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/VarianceSplitCalculatorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/VarianceSplitCalculatorTest.java
deleted file mode 100644
index d67cbc6..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/VarianceSplitCalculatorTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.VarianceSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-import org.junit.Test;
-
-/**
- * Test for {@link VarianceSplitCalculator}.
- */
-public class VarianceSplitCalculatorTest {
- /** Test calculation of region info consisting from one point. */
- @Test
- public void testCalculateRegionInfoSimple() {
- double labels[] = new double[] {0.0};
-
- assert new VarianceSplitCalculator().calculateRegionInfo(DoubleStream.of(labels), 1).impurity() == 0.0;
- }
-
- /** Test calculation of region info consisting from two classes. */
- @Test
- public void testCalculateRegionInfoTwoClasses() {
- double labels[] = new double[] {0.0, 1.0};
-
- assert new VarianceSplitCalculator().calculateRegionInfo(DoubleStream.of(labels), 2).impurity() == 0.25;
- }
-
- /** Test calculation of region info consisting from three classes. */
- @Test
- public void testCalculateRegionInfoThreeClasses() {
- double labels[] = new double[] {1.0, 2.0, 3.0};
-
- assert Math.abs(new VarianceSplitCalculator().calculateRegionInfo(DoubleStream.of(labels), 3).impurity() - 2.0 / 3) < 1E-10;
- }
-
- /** Test calculation of split of region consisting from one point. */
- @Test
- public void testSplitSimple() {
- double labels[] = new double[] {0.0};
- double values[] = new double[] {0.0};
- Integer[] samples = new Integer[] {0};
-
- VarianceSplitCalculator.VarianceData data = new VarianceSplitCalculator.VarianceData(0.0, 1, 0.0);
-
- assert new VarianceSplitCalculator().splitRegion(samples, values, labels, 0, data) == null;
- }
-
- /** Test calculation of split of region consisting from two classes. */
- @Test
- public void testSplitTwoClassesTwoPoints() {
- double labels[] = new double[] {0.0, 1.0};
- double values[] = new double[] {0.0, 1.0};
- Integer[] samples = new Integer[] {0, 1};
-
- VarianceSplitCalculator.VarianceData data = new VarianceSplitCalculator.VarianceData(0.25, 2, 0.5);
-
- SplitInfo<VarianceSplitCalculator.VarianceData> split = new VarianceSplitCalculator().splitRegion(samples, values, labels, 0, data);
-
- assert split.leftData().impurity() == 0;
- assert split.leftData().mean() == 0;
- assert split.leftData().getSize() == 1;
-
- assert split.rightData().impurity() == 0;
- assert split.rightData().mean() == 1;
- assert split.rightData().getSize() == 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/trees/performance/ColumnDecisionTreeTrainerBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/performance/ColumnDecisionTreeTrainerBenchmark.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/performance/ColumnDecisionTreeTrainerBenchmark.java
deleted file mode 100644
index 21fd692..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/performance/ColumnDecisionTreeTrainerBenchmark.java
+++ /dev/null
@@ -1,456 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.performance;
-
-import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Random;
-import java.util.UUID;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.estimators.Estimators;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Tracer;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distributed.keys.impl.SparseMatrixKey;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteTriFunction;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledVectorDouble;
-import org.apache.ignite.ml.trees.BaseDecisionTreeTest;
-import org.apache.ignite.ml.trees.SplitDataGenerator;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.trainers.columnbased.BiIndex;
-import org.apache.ignite.ml.trees.trainers.columnbased.BiIndexedCacheColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.MatrixColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ContextCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ProjectionsCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.SplitCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.GiniSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.VarianceSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-import org.apache.ignite.ml.util.MnistUtils;
-import org.apache.ignite.stream.StreamTransformer;
-import org.apache.ignite.testframework.junits.IgniteTestResources;
-import org.apache.log4j.Level;
-import org.junit.Assert;
-
-/**
- * Various benchmarks for hand runs.
- */
-public class ColumnDecisionTreeTrainerBenchmark extends BaseDecisionTreeTest {
- /** Name of the property specifying path to training set images. */
- private static final String PROP_TRAINING_IMAGES = "mnist.training.images";
-
- /** Name of property specifying path to training set labels. */
- private static final String PROP_TRAINING_LABELS = "mnist.training.labels";
-
- /** Name of property specifying path to test set images. */
- private static final String PROP_TEST_IMAGES = "mnist.test.images";
-
- /** Name of property specifying path to test set labels. */
- private static final String PROP_TEST_LABELS = "mnist.test.labels";
-
- /** Function to approximate. */
- private static final Function<Vector, Double> f1 = v -> v.get(0) * v.get(0) + 2 * Math.sin(v.get(1)) + v.get(2);
-
- /** {@inheritDoc} */
- @Override protected long getTestTimeout() {
- return 6000000;
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName,
- IgniteTestResources rsrcs) throws Exception {
- IgniteConfiguration configuration = super.getConfiguration(igniteInstanceName, rsrcs);
- // We do not need any extra event types.
- configuration.setIncludeEventTypes();
- configuration.setPeerClassLoadingEnabled(false);
-
- resetLog4j(Level.INFO, false, GridCacheProcessor.class.getPackage().getName());
-
- return configuration;
- }
-
- /**
- * This test is for manual run only.
- * To run this test rename this method so it starts from 'test'.
- */
- public void tstCacheMixed() {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- int ptsPerReg = 150;
- int featCnt = 10;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
- catsInfo.put(1, 3);
-
- Random rnd = new Random(12349L);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
- split(0, 1, new int[] {0, 2}).
- split(1, 0, -10.0).
- split(0, 0, 0.0);
-
- testByGenStreamerLoad(ptsPerReg, catsInfo, gen, rnd);
- }
-
- /**
- * Run decision tree classifier on MNIST using bi-indexed cache as a storage for dataset.
- * To run this test rename this method so it starts from 'test'.
- *
- * @throws IOException In case of loading MNIST dataset errors.
- */
- public void tstMNISTBiIndexedCache() throws IOException {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
- int ptsCnt = 40_000;
- int featCnt = 28 * 28;
-
- Properties props = loadMNISTProperties();
-
- Stream<DenseLocalOnHeapVector> trainingMnistStream = MnistUtils.mnistAsStream(props.getProperty(PROP_TRAINING_IMAGES), props.getProperty(PROP_TRAINING_LABELS), new Random(123L), ptsCnt);
- Stream<DenseLocalOnHeapVector> testMnistStream = MnistUtils.mnistAsStream(props.getProperty(PROP_TEST_IMAGES), props.getProperty(PROP_TEST_LABELS), new Random(123L), 10_000);
-
- IgniteCache<BiIndex, Double> cache = createBiIndexedCache();
-
- loadVectorsIntoBiIndexedCache(cache.getName(), trainingMnistStream.iterator(), featCnt + 1);
-
- ColumnDecisionTreeTrainer<GiniSplitCalculator.GiniData> trainer =
- new ColumnDecisionTreeTrainer<>(10, ContinuousSplitCalculators.GINI.apply(ignite), RegionCalculators.GINI, RegionCalculators.MOST_COMMON, ignite);
-
- X.println("Training started.");
- long before = System.currentTimeMillis();
- DecisionTreeModel mdl = trainer.train(new BiIndexedCacheColumnDecisionTreeTrainerInput(cache, new HashMap<>(), ptsCnt, featCnt));
- X.println("Training finished in " + (System.currentTimeMillis() - before));
-
- IgniteTriFunction<Model<Vector, Double>, Stream<IgniteBiTuple<Vector, Double>>, Function<Double, Double>, Double> mse = Estimators.errorsPercentage();
- Double accuracy = mse.apply(mdl, testMnistStream.map(v -> new IgniteBiTuple<>(v.viewPart(0, featCnt), v.getX(featCnt))), Function.identity());
- X.println("Errors percentage: " + accuracy);
-
- Assert.assertEquals(0, SplitCache.getOrCreate(ignite).size());
- Assert.assertEquals(0, FeaturesCache.getOrCreate(ignite).size());
- Assert.assertEquals(0, ContextCache.getOrCreate(ignite).size());
- Assert.assertEquals(0, ProjectionsCache.getOrCreate(ignite).size());
- }
-
- /**
- * Run decision tree classifier on MNIST using sparse distributed matrix as a storage for dataset.
- * To run this test rename this method so it starts from 'test'.
- *
- * @throws IOException In case of loading MNIST dataset errors.
- */
- public void tstMNISTSparseDistributedMatrix() throws IOException {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
- int ptsCnt = 30_000;
- int featCnt = 28 * 28;
-
- Properties props = loadMNISTProperties();
-
- Stream<DenseLocalOnHeapVector> trainingMnistStream = MnistUtils.mnistAsStream(props.getProperty(PROP_TRAINING_IMAGES), props.getProperty(PROP_TRAINING_LABELS), new Random(123L), ptsCnt);
- Stream<DenseLocalOnHeapVector> testMnistStream = MnistUtils.mnistAsStream(props.getProperty(PROP_TEST_IMAGES), props.getProperty(PROP_TEST_LABELS), new Random(123L), 10_000);
-
- SparseDistributedMatrix m = new SparseDistributedMatrix(ptsCnt, featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
- SparseDistributedMatrixStorage sto = (SparseDistributedMatrixStorage)m.getStorage();
-
- loadVectorsIntoSparseDistributedMatrixCache(sto.cache().getName(), sto.getUUID(), trainingMnistStream.iterator(), featCnt + 1);
-
- ColumnDecisionTreeTrainer<GiniSplitCalculator.GiniData> trainer =
- new ColumnDecisionTreeTrainer<>(10, ContinuousSplitCalculators.GINI.apply(ignite), RegionCalculators.GINI, RegionCalculators.MOST_COMMON, ignite);
-
- X.println("Training started");
- long before = System.currentTimeMillis();
- DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, new HashMap<>()));
- X.println("Training finished in " + (System.currentTimeMillis() - before));
-
- IgniteTriFunction<Model<Vector, Double>, Stream<IgniteBiTuple<Vector, Double>>, Function<Double, Double>, Double> mse = Estimators.errorsPercentage();
- Double accuracy = mse.apply(mdl, testMnistStream.map(v -> new IgniteBiTuple<>(v.viewPart(0, featCnt), v.getX(featCnt))), Function.identity());
- X.println("Errors percentage: " + accuracy);
-
- Assert.assertEquals(0, SplitCache.getOrCreate(ignite).size());
- Assert.assertEquals(0, FeaturesCache.getOrCreate(ignite).size());
- Assert.assertEquals(0, ContextCache.getOrCreate(ignite).size());
- Assert.assertEquals(0, ProjectionsCache.getOrCreate(ignite).size());
- }
-
- /** Load properties for MNIST tests. */
- private static Properties loadMNISTProperties() throws IOException {
- Properties res = new Properties();
-
- InputStream is = ColumnDecisionTreeTrainerBenchmark.class.getClassLoader().getResourceAsStream("manualrun/trees/columntrees.manualrun.properties");
-
- res.load(is);
-
- return res;
- }
-
- /** */
- private void testByGenStreamerLoad(int ptsPerReg, HashMap<Integer, Integer> catsInfo,
- SplitDataGenerator<DenseLocalOnHeapVector> gen, Random rnd) {
-
- List<IgniteBiTuple<Integer, DenseLocalOnHeapVector>> lst = gen.
- points(ptsPerReg, (i, rn) -> i).
- collect(Collectors.toList());
-
- int featCnt = gen.featuresCnt();
-
- Collections.shuffle(lst, rnd);
-
- int numRegs = gen.regsCount();
-
- SparseDistributedMatrix m = new SparseDistributedMatrix(numRegs * ptsPerReg, featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
- IgniteFunction<DoubleStream, Double> regCalc = s -> s.average().orElse(0.0);
-
- Map<Integer, List<LabeledVectorDouble>> byRegion = new HashMap<>();
-
- SparseDistributedMatrixStorage sto = (SparseDistributedMatrixStorage)m.getStorage();
- long before = System.currentTimeMillis();
- X.println("Batch loading started...");
- loadVectorsIntoSparseDistributedMatrixCache(sto.cache().getName(), sto.getUUID(), gen.
- points(ptsPerReg, (i, rn) -> i).map(IgniteBiTuple::get2).iterator(), featCnt + 1);
- X.println("Batch loading took " + (System.currentTimeMillis() - before) + " ms.");
-
- for (IgniteBiTuple<Integer, DenseLocalOnHeapVector> bt : lst) {
- byRegion.putIfAbsent(bt.get1(), new LinkedList<>());
- byRegion.get(bt.get1()).add(asLabeledVector(bt.get2().getStorage().data()));
- }
-
- ColumnDecisionTreeTrainer<VarianceSplitCalculator.VarianceData> trainer =
- new ColumnDecisionTreeTrainer<>(2, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, regCalc, ignite);
-
- before = System.currentTimeMillis();
- DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, catsInfo));
-
- X.println("Training took: " + (System.currentTimeMillis() - before) + " ms.");
-
- byRegion.keySet().forEach(k -> {
- LabeledVectorDouble sp = byRegion.get(k).get(0);
- Tracer.showAscii(sp.features());
- X.println("Predicted value and label [pred=" + mdl.apply(sp.features()) + ", label=" + sp.doubleLabel() + "]");
- assert mdl.apply(sp.features()) == sp.doubleLabel();
- });
- }
-
- /**
- * Test decision tree regression.
- * To run this test rename this method so it starts from 'test'.
- */
- public void tstF1() {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- int ptsCnt = 10000;
- Map<Integer, double[]> ranges = new HashMap<>();
-
- ranges.put(0, new double[] {-100.0, 100.0});
- ranges.put(1, new double[] {-100.0, 100.0});
- ranges.put(2, new double[] {-100.0, 100.0});
-
- int featCnt = 100;
- double[] defRng = {-1.0, 1.0};
-
- Vector[] trainVectors = vecsFromRanges(ranges, featCnt, defRng, new Random(123L), ptsCnt, f1);
-
- SparseDistributedMatrix m = new SparseDistributedMatrix(ptsCnt, featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
- SparseDistributedMatrixStorage sto = (SparseDistributedMatrixStorage)m.getStorage();
-
- loadVectorsIntoSparseDistributedMatrixCache(sto.cache().getName(), sto.getUUID(), Arrays.stream(trainVectors).iterator(), featCnt + 1);
-
- IgniteFunction<DoubleStream, Double> regCalc = s -> s.average().orElse(0.0);
-
- ColumnDecisionTreeTrainer<VarianceSplitCalculator.VarianceData> trainer =
- new ColumnDecisionTreeTrainer<>(10, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, regCalc, ignite);
-
- X.println("Training started.");
- long before = System.currentTimeMillis();
- DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, new HashMap<>()));
- X.println("Training finished in: " + (System.currentTimeMillis() - before) + " ms.");
-
- Vector[] testVectors = vecsFromRanges(ranges, featCnt, defRng, new Random(123L), 20, f1);
-
- IgniteTriFunction<Model<Vector, Double>, Stream<IgniteBiTuple<Vector, Double>>, Function<Double, Double>, Double> mse = Estimators.MSE();
- Double accuracy = mse.apply(mdl, Arrays.stream(testVectors).map(v -> new IgniteBiTuple<>(v.viewPart(0, featCnt), v.getX(featCnt))), Function.identity());
- X.println("MSE: " + accuracy);
- }
-
- /**
- * Load vectors into sparse distributed matrix.
- *
- * @param cacheName Name of cache where matrix is stored.
- * @param uuid UUID of matrix.
- * @param iter Iterator over vectors.
- * @param vectorSize size of vectors.
- */
- private void loadVectorsIntoSparseDistributedMatrixCache(String cacheName, UUID uuid,
- Iterator<? extends org.apache.ignite.ml.math.Vector> iter, int vectorSize) {
- try (IgniteDataStreamer<SparseMatrixKey, Map<Integer, Double>> streamer =
- Ignition.localIgnite().dataStreamer(cacheName)) {
- int sampleIdx = 0;
- streamer.allowOverwrite(true);
-
- streamer.receiver(StreamTransformer.from((e, arg) -> {
- Map<Integer, Double> val = e.getValue();
-
- if (val == null)
- val = new Int2DoubleOpenHashMap();
-
- val.putAll((Map<Integer, Double>)arg[0]);
-
- e.setValue(val);
-
- return null;
- }));
-
- // Feature index -> (sample index -> value)
- Map<Integer, Map<Integer, Double>> batch = new HashMap<>();
- IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
- int batchSize = 1000;
-
- while (iter.hasNext()) {
- org.apache.ignite.ml.math.Vector next = iter.next();
-
- for (int i = 0; i < vectorSize; i++)
- batch.get(i).put(sampleIdx, next.getX(i));
-
- X.println("Sample index: " + sampleIdx);
- if (sampleIdx % batchSize == 0) {
- batch.keySet().forEach(fi -> streamer.addData(new SparseMatrixKey(fi, uuid, fi), batch.get(fi)));
- IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
- }
- sampleIdx++;
- }
- if (sampleIdx % batchSize != 0) {
- batch.keySet().forEach(fi -> streamer.addData(new SparseMatrixKey(fi, uuid, fi), batch.get(fi)));
- IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
- }
- }
- }
-
- /**
- * Load vectors into bi-indexed cache.
- *
- * @param cacheName Name of cache.
- * @param iter Iterator over vectors.
- * @param vectorSize size of vectors.
- */
- private void loadVectorsIntoBiIndexedCache(String cacheName,
- Iterator<? extends org.apache.ignite.ml.math.Vector> iter, int vectorSize) {
- try (IgniteDataStreamer<BiIndex, Double> streamer =
- Ignition.localIgnite().dataStreamer(cacheName)) {
- int sampleIdx = 0;
-
- streamer.perNodeBufferSize(10000);
-
- while (iter.hasNext()) {
- org.apache.ignite.ml.math.Vector next = iter.next();
-
- for (int i = 0; i < vectorSize; i++)
- streamer.addData(new BiIndex(sampleIdx, i), next.getX(i));
-
- sampleIdx++;
-
- if (sampleIdx % 1000 == 0)
- System.out.println("Loaded: " + sampleIdx + " vectors.");
- }
- }
- }
-
- /**
- * Create bi-indexed cache for tests.
- *
- * @return Bi-indexed cache.
- */
- private IgniteCache<BiIndex, Double> createBiIndexedCache() {
- CacheConfiguration<BiIndex, Double> cfg = new CacheConfiguration<>();
-
- // Write to primary.
- cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
- // Atomic transactions only.
- cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
- // No eviction.
- cfg.setEvictionPolicy(null);
-
- // No copying of values.
- cfg.setCopyOnRead(false);
-
- // Cache is partitioned.
- cfg.setCacheMode(CacheMode.PARTITIONED);
-
- cfg.setBackups(0);
-
- cfg.setName("TMP_BI_INDEXED_CACHE");
-
- return Ignition.localIgnite().getOrCreateCache(cfg);
- }
-
- /** */
- private Vector[] vecsFromRanges(Map<Integer, double[]> ranges, int featCnt, double[] defRng, Random rnd, int ptsCnt,
- Function<Vector, Double> f) {
- int vs = featCnt + 1;
- DenseLocalOnHeapVector[] res = new DenseLocalOnHeapVector[ptsCnt];
- for (int pt = 0; pt < ptsCnt; pt++) {
- DenseLocalOnHeapVector v = new DenseLocalOnHeapVector(vs);
- for (int i = 0; i < featCnt; i++) {
- double[] range = ranges.getOrDefault(i, defRng);
- double from = range[0];
- double to = range[1];
- double rng = to - from;
-
- v.setX(i, rnd.nextDouble() * rng);
- }
- v.setX(featCnt, f.apply(v));
- res[pt] = v;
- }
-
- return res;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
deleted file mode 100644
index f8a7c08..0000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.trees;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.thread.IgniteThread;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteColumnDecisionTreeGiniBenchmark extends IgniteAbstractBenchmark {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public boolean test(Map<Object, Object> ctx) throws Exception {
- // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
- // because we create ignite cache internally.
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- this.getClass().getSimpleName(), new Runnable() {
- /** {@inheritDoc} */
- @Override public void run() {
- // IMPL NOTE originally taken from ColumnDecisionTreeTrainerTest#testCacheMixedGini
- int totalPts = 1 << 10;
- int featCnt = 2;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
- catsInfo.put(1, 3);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1)).
- split(0, 1, new int[] {0, 2}).
- split(1, 0, -10.0);
-
- gen.testByGen(totalPts, ContinuousSplitCalculators.GINI.apply(ignite),
- RegionCalculators.GINI, RegionCalculators.MEAN, ignite);
- }
- });
-
- igniteThread.start();
-
- igniteThread.join();
-
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
deleted file mode 100644
index f9d417f..0000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.trees;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.thread.IgniteThread;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteColumnDecisionTreeVarianceBenchmark extends IgniteAbstractBenchmark {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public boolean test(Map<Object, Object> ctx) throws Exception {
- // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
- // because we create ignite cache internally.
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- this.getClass().getSimpleName(), new Runnable() {
- /** {@inheritDoc} */
- @Override public void run() {
- // IMPL NOTE originally taken from ColumnDecisionTreeTrainerTest#testCacheMixed
- int totalPts = 1 << 10;
- int featCnt = 2;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
- catsInfo.put(1, 3);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen
- = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1)).
- split(0, 1, new int[] {0, 2}).
- split(1, 0, -10.0);
-
- gen.testByGen(totalPts,
- ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, RegionCalculators.MEAN, ignite);
- }
- });
-
- igniteThread.start();
-
- igniteThread.join();
-
- return true;
- }
-}
[12/14] ignite git commit: IGNITE-8101 Ability to terminate system
workers by JMX for test purposes.
Posted by ak...@apache.org.
IGNITE-8101 Ability to terminate system workers by JMX for test purposes.
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a96ac047
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a96ac047
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a96ac047
Branch: refs/heads/ignite-8201
Commit: a96ac04755d2c7448508990fe9e1dad62bcccc90
Parents: 3a71765
Author: Dmitriy Sorokin <d....@gmail.com>
Authored: Tue Apr 10 22:20:41 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Apr 10 22:20:41 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 7 ++
.../failure/StopNodeOrHaltFailureHandler.java | 2 +-
.../ignite/internal/GridKernalContext.java | 8 ++
.../ignite/internal/GridKernalContextImpl.java | 10 +++
.../apache/ignite/internal/IgniteKernal.java | 16 +++-
.../discovery/GridDiscoveryManager.java | 2 +-
.../GridCachePartitionExchangeManager.java | 3 +-
.../cache/GridCacheSharedTtlCleanupManager.java | 3 +-
.../wal/reader/StandaloneGridKernalContext.java | 6 ++
.../timeout/GridTimeoutProcessor.java | 3 +-
.../ignite/internal/util/IgniteUtils.java | 7 +-
.../worker/WorkersControlMXBeanImpl.java | 62 +++++++++++++++
.../ignite/internal/worker/WorkersRegistry.java | 80 ++++++++++++++++++++
.../ignite/internal/worker/package-info.java | 22 ++++++
.../ignite/mxbean/WorkersControlMXBean.java | 49 ++++++++++++
15 files changed, 271 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 152d845..9da123e 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -423,6 +423,13 @@ public final class IgniteSystemProperties {
public static final String IGNITE_MBEANS_DISABLED = "IGNITE_MBEANS_DISABLED";
/**
+ * If property is set to {@code true}, then test features will be enabled.
+ *
+ * Default is {@code false}.
+ */
+ public static final String IGNITE_TEST_FEATURES_ENABLED = "IGNITE_TEST_FEATURES_ENABLED";
+
+ /**
* Property controlling size of buffer holding last exception. Default value of {@code 1000}.
*/
public static final String IGNITE_EXCEPTION_REGISTRY_MAX_SIZE = "IGNITE_EXCEPTION_REGISTRY_MAX_SIZE";
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
index 4f74406..3ce4ff6 100644
--- a/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
@@ -92,7 +92,7 @@ public class StopNodeOrHaltFailureHandler implements FailureHandler {
).start();
}
else {
- U.error(log, "JVM will be halted immediately on ignite failure: [failureCtx=" + failureCtx + ']');
+ U.error(log, "JVM will be halted immediately due to the failure: [failureCtx=" + failureCtx + ']');
Runtime.getRuntime().halt(Ignition.KILL_EXIT_CODE);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index 0b40054..505c3d6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -32,6 +32,7 @@ import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
import org.apache.ignite.internal.managers.failover.GridFailoverManager;
import org.apache.ignite.internal.managers.indexing.GridIndexingManager;
import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager;
+import org.apache.ignite.internal.worker.WorkersRegistry;
import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor;
import org.apache.ignite.internal.processors.authentication.IgniteAuthenticationProcessor;
import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
@@ -423,6 +424,13 @@ public interface GridKernalContext extends Iterable<GridComponent> {
public GridIndexingManager indexing();
/**
+ * Gets workers registry.
+ *
+ * @return Workers registry.
+ */
+ public WorkersRegistry workersRegistry();
+
+ /**
* Gets data structures processor.
*
* @return Data structures processor.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 3408334..ac49708 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -46,6 +46,7 @@ import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
import org.apache.ignite.internal.managers.failover.GridFailoverManager;
import org.apache.ignite.internal.managers.indexing.GridIndexingManager;
import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager;
+import org.apache.ignite.internal.worker.WorkersRegistry;
import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor;
import org.apache.ignite.internal.processors.authentication.IgniteAuthenticationProcessor;
import org.apache.ignite.internal.processors.cache.CacheConflictResolutionManager;
@@ -361,6 +362,10 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
private Map<String, Object> attrs = new HashMap<>();
/** */
+ @GridToStringExclude
+ private final WorkersRegistry workersRegistry = new WorkersRegistry();
+
+ /** */
private IgniteEx grid;
/** */
@@ -780,6 +785,11 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
}
/** {@inheritDoc} */
+ @Override public WorkersRegistry workersRegistry() {
+ return workersRegistry;
+ }
+
+ /** {@inheritDoc} */
@Override public GridAffinityProcessor affinity() {
return affProc;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 0b102e5..1cb07b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -125,7 +125,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
import org.apache.ignite.internal.processors.cache.GridCacheUtilityKey;
-import org.apache.ignite.internal.processors.cache.GridCacheUtils;
import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
@@ -185,6 +184,8 @@ import org.apache.ignite.internal.util.typedef.internal.LT;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.SB;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.worker.WorkersControlMXBeanImpl;
+import org.apache.ignite.internal.worker.WorkersRegistry;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgnitePredicate;
@@ -197,6 +198,7 @@ import org.apache.ignite.marshaller.jdk.JdkMarshaller;
import org.apache.ignite.mxbean.ClusterMetricsMXBean;
import org.apache.ignite.mxbean.IgniteMXBean;
import org.apache.ignite.mxbean.StripedExecutorMXBean;
+import org.apache.ignite.mxbean.WorkersControlMXBean;
import org.apache.ignite.mxbean.ThreadPoolMXBean;
import org.apache.ignite.plugin.IgnitePlugin;
import org.apache.ignite.plugin.PluginNotFoundException;
@@ -1085,7 +1087,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
// Register MBeans.
mBeansMgr.registerAllMBeans(utilityCachePool, execSvc, svcExecSvc, sysExecSvc, stripedExecSvc, p2pExecSvc,
mgmtExecSvc, igfsExecSvc, dataStreamExecSvc, restExecSvc, affExecSvc, idxExecSvc, callbackExecSvc,
- qryExecSvc, schemaExecSvc, customExecSvcs);
+ qryExecSvc, schemaExecSvc, customExecSvcs, ctx.workersRegistry());
// Lifecycle bean notifications.
notifyLifecycleBeans(AFTER_NODE_START);
@@ -4175,7 +4177,8 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
IgniteStripedThreadPoolExecutor callbackExecSvc,
ExecutorService qryExecSvc,
ExecutorService schemaExecSvc,
- @Nullable final Map<String, ? extends ExecutorService> customExecSvcs
+ @Nullable final Map<String, ? extends ExecutorService> customExecSvcs,
+ WorkersRegistry workersRegistry
) throws IgniteCheckedException {
if (U.IGNITE_MBEANS_DISABLED)
return;
@@ -4221,6 +4224,13 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
for (Map.Entry<String, ? extends ExecutorService> entry : customExecSvcs.entrySet())
registerExecutorMBean(entry.getKey(), entry.getValue());
}
+
+ if (U.IGNITE_TEST_FEATURES_ENABLED) {
+ WorkersControlMXBean workerCtrlMXBean = new WorkersControlMXBeanImpl(workersRegistry);
+
+ registerMBean("Kernal", workerCtrlMXBean.getClass().getSimpleName(),
+ workerCtrlMXBean, WorkersControlMXBean.class);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index b0d3256..a1d84e5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -2591,7 +2591,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
*
*/
private DiscoveryWorker() {
- super(ctx.igniteInstanceName(), "disco-event-worker", GridDiscoveryManager.this.log);
+ super(ctx.igniteInstanceName(), "disco-event-worker", GridDiscoveryManager.this.log, ctx.workersRegistry());
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index e40493f..1a0e65f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -2077,7 +2077,8 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
* Constructor.
*/
private ExchangeWorker() {
- super(cctx.igniteInstanceName(), "partition-exchanger", GridCachePartitionExchangeManager.this.log);
+ super(cctx.igniteInstanceName(), "partition-exchanger", GridCachePartitionExchangeManager.this.log,
+ cctx.kernalContext().workersRegistry());
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
index 613e93b..7adabc3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
@@ -121,7 +121,8 @@ public class GridCacheSharedTtlCleanupManager extends GridCacheSharedManagerAdap
* Creates cleanup worker.
*/
CleanupWorker() {
- super(cctx.igniteInstanceName(), "ttl-cleanup-worker", cctx.logger(GridCacheSharedTtlCleanupManager.class));
+ super(cctx.igniteInstanceName(), "ttl-cleanup-worker", cctx.logger(GridCacheSharedTtlCleanupManager.class),
+ cctx.kernalContext().workersRegistry());
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index 429a5ce..cb04575 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -86,6 +86,7 @@ import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
import org.apache.ignite.internal.util.IgniteExceptionRegistry;
import org.apache.ignite.internal.util.StripedExecutor;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.worker.WorkersRegistry;
import org.apache.ignite.marshaller.Marshaller;
import org.apache.ignite.plugin.PluginNotFoundException;
import org.apache.ignite.plugin.PluginProvider;
@@ -454,6 +455,11 @@ public class StandaloneGridKernalContext implements GridKernalContext {
}
/** {@inheritDoc} */
+ @Override public WorkersRegistry workersRegistry() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
@Override public DataStructuresProcessor dataStructures() {
return null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
index a09d6fa..25151cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
@@ -145,7 +145,8 @@ public class GridTimeoutProcessor extends GridProcessorAdapter {
*
*/
TimeoutWorker() {
- super(ctx.config().getIgniteInstanceName(), "grid-timeout-worker", GridTimeoutProcessor.this.log);
+ super(ctx.config().getIgniteInstanceName(), "grid-timeout-worker",
+ GridTimeoutProcessor.this.log, ctx.workersRegistry());
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 93f4fb4..42e96fb 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -522,7 +522,12 @@ public abstract class IgniteUtils {
};
/** Ignite MBeans disabled flag. */
- public static boolean IGNITE_MBEANS_DISABLED = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_MBEANS_DISABLED);
+ public static boolean IGNITE_MBEANS_DISABLED =
+ IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_MBEANS_DISABLED);
+
+ /** Ignite test features enabled flag. */
+ public static boolean IGNITE_TEST_FEATURES_ENABLED =
+ IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_TEST_FEATURES_ENABLED);
/** */
private static final boolean assertionsEnabled;
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
new file mode 100644
index 0000000..9e427e8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.worker;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.mxbean.WorkersControlMXBean;
+
+/**
+ * MBean that provides control of system workersRegistry.
+ */
+public class WorkersControlMXBeanImpl implements WorkersControlMXBean {
+ /** System worker registry. */
+ private final WorkersRegistry workerRegistry;
+
+ /**
+ * Constructor.
+ *
+ * @param registry System worker registry.
+ */
+ public WorkersControlMXBeanImpl(WorkersRegistry registry) {
+ workerRegistry = registry;
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<String> getWorkerNames() {
+ return new ArrayList<>(workerRegistry.names());
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean terminateWorker(String name) {
+ GridWorker w = workerRegistry.worker(name);
+
+ if (w == null || w.isCancelled())
+ return false;
+
+ Thread t = w.runner();
+
+ if (t == null)
+ return false;
+
+ t.interrupt();
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
new file mode 100644
index 0000000..e8d46fb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.worker;
+
+import java.util.Collection;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.internal.util.worker.GridWorkerListener;
+
+/**
+ * Workers registry.
+ */
+public class WorkersRegistry implements GridWorkerListener {
+ /** Registered workers. */
+ private final ConcurrentMap<String, GridWorker> registeredWorkers = new ConcurrentHashMap<>();
+
+ /**
+ * Adds worker to the registry.
+ *
+ * @param w Worker.
+ */
+ public void register(GridWorker w) {
+ if (registeredWorkers.putIfAbsent(w.name(), w) != null)
+ throw new IllegalStateException("Worker is already registered [worker=" + w + ']');
+ }
+
+ /**
+ * Removes worker from the registry.
+ *
+ * @param name Worker name.
+ */
+ public void unregister(String name) {
+ registeredWorkers.remove(name);
+ }
+
+ /**
+ * Returns names of all registered workers.
+ *
+ * @return Registered worker names.
+ */
+ public Collection<String> names() {
+ return registeredWorkers.keySet();
+ }
+
+ /**
+ * Returns worker with given name.
+ *
+ * @param name Name.
+ * @return Registered {@link GridWorker} with name {@code name} or {@code null} if not found.
+ */
+ public GridWorker worker(String name) {
+ return registeredWorkers.get(name);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onStarted(GridWorker w) {
+ register(w);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onStopped(GridWorker w) {
+ unregister(w.name());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java
new file mode 100644
index 0000000..03ca621
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * System worker registry and control MBean implementation.
+ */
+package org.apache.ignite.internal.worker;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java
new file mode 100644
index 0000000..0f5419b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.mxbean;
+
+import java.util.List;
+
+/**
+ * MBean that provides ability to terminate worker that registered in the workers registry.
+ */
+@MXBeanDescription("MBean that provides ability to terminate worker that registered in the workers registry.")
+public interface WorkersControlMXBean {
+ /**
+ * Returns names of all registered workers.
+ *
+ * @return Worker names.
+ */
+ @MXBeanDescription("Names of registered workers.")
+ public List<String> getWorkerNames();
+
+ /**
+ * Terminates worker.
+ *
+ * @param name Worker name.
+ * @return {@code True} if worker has been terminated successfully, {@code false} otherwise.
+ */
+ @MXBeanDescription("Terminates worker.")
+ @MXBeanParametersNames(
+ "name"
+ )
+ @MXBeanParametersDescriptions(
+ "Name of worker to terminate."
+ )
+ public boolean terminateWorker(String name);
+}
[07/14] ignite git commit: IGNITE-7927 Web Console: Fixed demo for
non-collocated joins.
Posted by ak...@apache.org.
IGNITE-7927 Web Console: Fixed demo for non-collocated joins.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/647620b3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/647620b3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/647620b3
Branch: refs/heads/ignite-8201
Commit: 647620b3ccf204c7bf11c4ff540584d0c99d9a15
Parents: 139c2af
Author: Vasiliy Sisko <vs...@gridgain.com>
Authored: Tue Apr 10 17:48:52 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Apr 10 17:48:52 2018 +0700
----------------------------------------------------------------------
modules/web-console/backend/routes/demo.js | 2 ++
.../demo/service/DemoCachesLoadService.java | 22 ++++++++++++++++++--
2 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/647620b3/modules/web-console/backend/routes/demo.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/routes/demo.js b/modules/web-console/backend/routes/demo.js
index a18fa7a..b081d0c 100644
--- a/modules/web-console/backend/routes/demo.js
+++ b/modules/web-console/backend/routes/demo.js
@@ -95,10 +95,12 @@ module.exports.factory = (errors, settings, mongo, spacesService) => {
domain.space = cacheDoc.space;
domain.caches.push(cacheDoc._id);
+ domain.clusters.push(cluster._id);
return domain.save()
.then((domainDoc) => {
cacheDoc.domains.push(domainDoc._id);
+ cluster.models.push(domainDoc._id);
return cacheDoc.save();
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/647620b3/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
----------------------------------------------------------------------
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
index 6691d1d..2aace06 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
@@ -269,6 +269,14 @@ public class DemoCachesLoadService implements Service {
type.setFields(qryFlds);
+ // Indexes for DEPARTMENT.
+
+ ArrayList<QueryIndex> indexes = new ArrayList<>();
+
+ indexes.add(new QueryIndex("countryId", QueryIndexType.SORTED, false, "DEP_COUNTRY"));
+
+ type.setIndexes(indexes);
+
ccfg.setQueryEntities(qryEntities);
return ccfg;
@@ -312,6 +320,11 @@ public class DemoCachesLoadService implements Service {
// Indexes for EMPLOYEE.
+ Collection<QueryIndex> indexes = new ArrayList<>();
+
+ indexes.add(new QueryIndex("departmentId", QueryIndexType.SORTED, false, "EMP_DEPARTMENT"));
+ indexes.add(new QueryIndex("managerId", QueryIndexType.SORTED, false, "EMP_MANAGER"));
+
QueryIndex idx = new QueryIndex();
idx.setName("EMP_NAMES");
@@ -323,8 +336,6 @@ public class DemoCachesLoadService implements Service {
idx.setFields(indFlds);
- Collection<QueryIndex> indexes = new ArrayList<>();
-
indexes.add(idx);
indexes.add(new QueryIndex("salary", QueryIndexType.SORTED, false, "EMP_SALARY"));
@@ -392,6 +403,13 @@ public class DemoCachesLoadService implements Service {
type.setFields(qryFlds);
+ // Indexes for CAR.
+
+ ArrayList<QueryIndex> indexes = new ArrayList<>();
+
+ indexes.add(new QueryIndex("parkingId", QueryIndexType.SORTED, false, "CAR_PARKING"));
+ type.setIndexes(indexes);
+
ccfg.setQueryEntities(qryEntities);
return ccfg;
[08/14] ignite git commit: IGNITE-8025 Future must fail if assertion
error has been thrown in the worker thread
Posted by ak...@apache.org.
IGNITE-8025 Future must fail if assertion error has been thrown in the worker thread
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/05d7092e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/05d7092e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/05d7092e
Branch: refs/heads/ignite-8201
Commit: 05d7092e255295cf6b78b9d9082ad9eab5e8a246
Parents: 647620b
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Tue Apr 10 16:22:28 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Apr 10 16:23:06 2018 +0300
----------------------------------------------------------------------
.../test/java/org/apache/ignite/testframework/GridTestUtils.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d7092e/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index 4e9a7c2..e6c6657 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -814,7 +814,7 @@ public final class GridTestUtils {
catch (IgniteFutureCancelledCheckedException e) {
resFut.onCancelled();
}
- catch (IgniteCheckedException e) {
+ catch (Throwable e) {
resFut.onDone(e);
}
});
[06/14] ignite git commit: IGNITE-8059: Integrate decision tree with
partition based dataset.
Posted by ak...@apache.org.
IGNITE-8059: Integrate decision tree with partition based dataset.
this closes #3760
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/139c2af6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/139c2af6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/139c2af6
Branch: refs/heads/ignite-8201
Commit: 139c2af66a9f745f89429842810f5d5fe1addf28
Parents: a64b941
Author: dmitrievanthony <dm...@gmail.com>
Authored: Tue Apr 10 12:46:43 2018 +0300
Committer: YuriBabak <y....@gmail.com>
Committed: Tue Apr 10 12:46:44 2018 +0300
----------------------------------------------------------------------
...ecisionTreeClassificationTrainerExample.java | 147 +++++
.../DecisionTreeRegressionTrainerExample.java | 124 ++++
.../ignite/examples/ml/tree/package-info.java | 22 +
.../examples/ml/trees/DecisionTreesExample.java | 354 ------------
.../ignite/examples/ml/trees/package-info.java | 22 -
.../main/java/org/apache/ignite/ml/Trainer.java | 3 -
.../org/apache/ignite/ml/tree/DecisionTree.java | 252 ++++++++
.../tree/DecisionTreeClassificationTrainer.java | 93 +++
.../ml/tree/DecisionTreeConditionalNode.java | 78 +++
.../ignite/ml/tree/DecisionTreeLeafNode.java | 48 ++
.../apache/ignite/ml/tree/DecisionTreeNode.java | 26 +
.../ml/tree/DecisionTreeRegressionTrainer.java | 60 ++
.../org/apache/ignite/ml/tree/TreeFilter.java | 38 ++
.../ignite/ml/tree/data/DecisionTreeData.java | 128 +++++
.../ml/tree/data/DecisionTreeDataBuilder.java | 73 +++
.../ignite/ml/tree/data/package-info.java | 22 +
.../ml/tree/impurity/ImpurityMeasure.java | 55 ++
.../impurity/ImpurityMeasureCalculator.java | 38 ++
.../tree/impurity/gini/GiniImpurityMeasure.java | 115 ++++
.../gini/GiniImpurityMeasureCalculator.java | 110 ++++
.../ml/tree/impurity/gini/package-info.java | 22 +
.../tree/impurity/mse/MSEImpurityMeasure.java | 133 +++++
.../mse/MSEImpurityMeasureCalculator.java | 80 +++
.../ml/tree/impurity/mse/package-info.java | 22 +
.../ignite/ml/tree/impurity/package-info.java | 22 +
.../util/SimpleStepFunctionCompressor.java | 149 +++++
.../ml/tree/impurity/util/StepFunction.java | 162 ++++++
.../impurity/util/StepFunctionCompressor.java | 55 ++
.../ml/tree/impurity/util/package-info.java | 22 +
.../ml/tree/leaf/DecisionTreeLeafBuilder.java | 38 ++
.../tree/leaf/MeanDecisionTreeLeafBuilder.java | 73 +++
.../leaf/MostCommonDecisionTreeLeafBuilder.java | 86 +++
.../ignite/ml/tree/leaf/package-info.java | 22 +
.../org/apache/ignite/ml/tree/package-info.java | 22 +
.../ignite/ml/trees/CategoricalRegionInfo.java | 72 ---
.../ignite/ml/trees/CategoricalSplitInfo.java | 68 ---
.../ignite/ml/trees/ContinuousRegionInfo.java | 74 ---
.../ml/trees/ContinuousSplitCalculator.java | 51 --
.../org/apache/ignite/ml/trees/RegionInfo.java | 62 --
.../ml/trees/models/DecisionTreeModel.java | 44 --
.../ignite/ml/trees/models/package-info.java | 22 -
.../ml/trees/nodes/CategoricalSplitNode.java | 50 --
.../ml/trees/nodes/ContinuousSplitNode.java | 56 --
.../ignite/ml/trees/nodes/DecisionTreeNode.java | 33 --
.../org/apache/ignite/ml/trees/nodes/Leaf.java | 49 --
.../apache/ignite/ml/trees/nodes/SplitNode.java | 100 ----
.../ignite/ml/trees/nodes/package-info.java | 22 -
.../apache/ignite/ml/trees/package-info.java | 22 -
.../ml/trees/trainers/columnbased/BiIndex.java | 113 ----
...exedCacheColumnDecisionTreeTrainerInput.java | 57 --
.../CacheColumnDecisionTreeTrainerInput.java | 141 -----
.../columnbased/ColumnDecisionTreeTrainer.java | 568 -------------------
.../ColumnDecisionTreeTrainerInput.java | 55 --
.../MatrixColumnDecisionTreeTrainerInput.java | 83 ---
.../trainers/columnbased/RegionProjection.java | 109 ----
.../trainers/columnbased/TrainingContext.java | 166 ------
.../columnbased/caches/ContextCache.java | 68 ---
.../columnbased/caches/FeaturesCache.java | 151 -----
.../columnbased/caches/ProjectionsCache.java | 286 ----------
.../trainers/columnbased/caches/SplitCache.java | 206 -------
.../columnbased/caches/package-info.java | 22 -
.../ContinuousSplitCalculators.java | 34 --
.../contsplitcalcs/GiniSplitCalculator.java | 234 --------
.../contsplitcalcs/VarianceSplitCalculator.java | 179 ------
.../contsplitcalcs/package-info.java | 22 -
.../trainers/columnbased/package-info.java | 22 -
.../columnbased/regcalcs/RegionCalculators.java | 85 ---
.../columnbased/regcalcs/package-info.java | 22 -
.../vectors/CategoricalFeatureProcessor.java | 212 -------
.../vectors/ContinuousFeatureProcessor.java | 111 ----
.../vectors/ContinuousSplitInfo.java | 71 ---
.../columnbased/vectors/FeatureProcessor.java | 82 ---
.../vectors/FeatureVectorProcessorUtils.java | 57 --
.../columnbased/vectors/SampleInfo.java | 80 ---
.../trainers/columnbased/vectors/SplitInfo.java | 106 ----
.../columnbased/vectors/package-info.java | 22 -
.../org/apache/ignite/ml/IgniteMLTestSuite.java | 4 +-
.../ml/nn/performance/MnistMLPTestUtil.java | 9 +-
...reeClassificationTrainerIntegrationTest.java | 100 ++++
.../DecisionTreeClassificationTrainerTest.java | 91 +++
...ionTreeRegressionTrainerIntegrationTest.java | 100 ++++
.../tree/DecisionTreeRegressionTrainerTest.java | 91 +++
.../ignite/ml/tree/DecisionTreeTestSuite.java | 48 ++
.../ml/tree/data/DecisionTreeDataTest.java | 59 ++
.../gini/GiniImpurityMeasureCalculatorTest.java | 103 ++++
.../impurity/gini/GiniImpurityMeasureTest.java | 131 +++++
.../mse/MSEImpurityMeasureCalculatorTest.java | 59 ++
.../impurity/mse/MSEImpurityMeasureTest.java | 109 ++++
.../util/SimpleStepFunctionCompressorTest.java | 75 +++
.../ml/tree/impurity/util/StepFunctionTest.java | 71 +++
.../tree/impurity/util/TestImpurityMeasure.java | 88 +++
.../DecisionTreeMNISTIntegrationTest.java | 105 ++++
.../tree/performance/DecisionTreeMNISTTest.java | 74 +++
.../ignite/ml/trees/BaseDecisionTreeTest.java | 70 ---
.../ml/trees/ColumnDecisionTreeTrainerTest.java | 191 -------
.../ignite/ml/trees/DecisionTreesTestSuite.java | 33 --
.../ml/trees/GiniSplitCalculatorTest.java | 141 -----
.../ignite/ml/trees/SplitDataGenerator.java | 390 -------------
.../ml/trees/VarianceSplitCalculatorTest.java | 84 ---
.../ColumnDecisionTreeTrainerBenchmark.java | 456 ---------------
.../IgniteColumnDecisionTreeGiniBenchmark.java | 70 ---
...niteColumnDecisionTreeVarianceBenchmark.java | 71 ---
.../yardstick/ml/trees/SplitDataGenerator.java | 426 --------------
.../ignite/yardstick/ml/trees/package-info.java | 22 -
104 files changed, 3647 insertions(+), 6429 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
new file mode 100644
index 0000000..cef6368
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.tree;
+
+import java.util.Random;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
+import org.apache.ignite.ml.tree.DecisionTreeNode;
+import org.apache.ignite.thread.IgniteThread;
+
+/**
+ * Example of using distributed {@link DecisionTreeClassificationTrainer}.
+ */
+public class DecisionTreeClassificationTrainerExample {
+ /**
+ * Executes example.
+ *
+ * @param args Command line arguments, none required.
+ */
+ public static void main(String... args) throws InterruptedException {
+ System.out.println(">>> Decision tree classification trainer example started.");
+
+ // Start ignite grid.
+ try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+ System.out.println(">>> Ignite grid started.");
+
+ IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+ DecisionTreeClassificationTrainerExample.class.getSimpleName(), () -> {
+
+ // Create cache with training data.
+ CacheConfiguration<Integer, LabeledPoint> trainingSetCfg = new CacheConfiguration<>();
+ trainingSetCfg.setName("TRAINING_SET");
+ trainingSetCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+ IgniteCache<Integer, LabeledPoint> trainingSet = ignite.createCache(trainingSetCfg);
+
+ Random rnd = new Random(0);
+
+ // Fill training data.
+ for (int i = 0; i < 1000; i++)
+ trainingSet.put(i, generatePoint(rnd));
+
+ // Create classification trainer.
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(4, 0);
+
+ // Train decision tree model.
+ DecisionTreeNode mdl = trainer.fit(
+ new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+ (k, v) -> new double[]{v.x, v.y},
+ (k, v) -> v.lb
+ );
+
+ // Calculate score.
+ int correctPredictions = 0;
+ for (int i = 0; i < 1000; i++) {
+ LabeledPoint pnt = generatePoint(rnd);
+
+ double prediction = mdl.apply(new double[]{pnt.x, pnt.y});
+
+ if (prediction == pnt.lb)
+ correctPredictions++;
+ }
+
+ System.out.println(">>> Accuracy: " + correctPredictions / 10.0 + "%");
+
+ System.out.println(">>> Decision tree classification trainer example completed.");
+ });
+
+ igniteThread.start();
+
+ igniteThread.join();
+ }
+ }
+
+ /**
+ * Generate point with {@code x} in (-0.5, 0.5) and {@code y} in the same interval. If {@code x * y > 0} then label
+ * is 1, otherwise 0.
+ *
+ * @param rnd Random.
+ * @return Point with label.
+ */
+ private static LabeledPoint generatePoint(Random rnd) {
+
+ double x = rnd.nextDouble() - 0.5;
+ double y = rnd.nextDouble() - 0.5;
+
+ return new LabeledPoint(x, y, x * y > 0 ? 1 : 0);
+ }
+
+ /** Point data class. */
+ private static class Point {
+ /** X coordinate. */
+ final double x;
+
+ /** Y coordinate. */
+ final double y;
+
+ /**
+ * Constructs a new instance of point.
+ *
+ * @param x X coordinate.
+ * @param y Y coordinate.
+ */
+ Point(double x, double y) {
+ this.x = x;
+ this.y = y;
+ }
+ }
+
+ /** Labeled point data class. */
+ private static class LabeledPoint extends Point {
+ /** Point label. */
+ final double lb;
+
+ /**
+ * Constructs a new instance of labeled point data.
+ *
+ * @param x X coordinate.
+ * @param y Y coordinate.
+ * @param lb Point label.
+ */
+ LabeledPoint(double x, double y, double lb) {
+ super(x, y);
+ this.lb = lb;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
new file mode 100644
index 0000000..61ba5f9
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.tree;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.tree.DecisionTreeNode;
+import org.apache.ignite.ml.tree.DecisionTreeRegressionTrainer;
+import org.apache.ignite.thread.IgniteThread;
+
+/**
+ * Example of using distributed {@link DecisionTreeRegressionTrainer}.
+ */
+public class DecisionTreeRegressionTrainerExample {
+ /**
+ * Executes example.
+ *
+ * @param args Command line arguments, none required.
+ */
+ public static void main(String... args) throws InterruptedException {
+ System.out.println(">>> Decision tree regression trainer example started.");
+
+ // Start ignite grid.
+ try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+ System.out.println(">>> Ignite grid started.");
+
+ IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+ DecisionTreeRegressionTrainerExample.class.getSimpleName(), () -> {
+
+ // Create cache with training data.
+ CacheConfiguration<Integer, Point> trainingSetCfg = new CacheConfiguration<>();
+ trainingSetCfg.setName("TRAINING_SET");
+ trainingSetCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+ IgniteCache<Integer, Point> trainingSet = ignite.createCache(trainingSetCfg);
+
+ // Fill training data.
+ generatePoints(trainingSet);
+
+ // Create regression trainer.
+ DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(10, 0);
+
+ // Train decision tree model.
+ DecisionTreeNode mdl = trainer.fit(
+ new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+ (k, v) -> new double[] {v.x},
+ (k, v) -> v.y
+ );
+
+ System.out.println(">>> Linear regression model: " + mdl);
+
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Ground Truth\t|");
+ System.out.println(">>> ---------------------------------");
+
+ // Calculate score.
+ for (int x = 0; x < 10; x++) {
+ double predicted = mdl.apply(new double[] {x});
+
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", predicted, Math.sin(x));
+ }
+
+ System.out.println(">>> ---------------------------------");
+
+ System.out.println(">>> Decision tree regression trainer example completed.");
+ });
+
+ igniteThread.start();
+
+ igniteThread.join();
+ }
+ }
+
+ /**
+ * Generates {@code sin(x)} on interval [0, 10) and loads into the specified cache.
+ */
+ private static void generatePoints(IgniteCache<Integer, Point> trainingSet) {
+ for (int i = 0; i < 1000; i++) {
+ double x = i / 100.0;
+ double y = Math.sin(x);
+
+ trainingSet.put(i, new Point(x, y));
+ }
+ }
+
+ /** Point data class. */
+ private static class Point {
+ /** X coordinate. */
+ final double x;
+
+ /** Y coordinate. */
+ final double y;
+
+ /**
+ * Constructs a new instance of point.
+ *
+ * @param x X coordinate.
+ * @param y Y coordinate.
+ */
+ Point(double x, double y) {
+ this.x = x;
+ this.y = y;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/examples/src/main/java/org/apache/ignite/examples/ml/tree/package-info.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/package-info.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/package-info.java
new file mode 100644
index 0000000..d8d9de6
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Decision trees examples.
+ */
+package org.apache.ignite.examples.ml.tree;
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/examples/src/main/java/org/apache/ignite/examples/ml/trees/DecisionTreesExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/trees/DecisionTreesExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/trees/DecisionTreesExample.java
deleted file mode 100644
index b1b2c42..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/trees/DecisionTreesExample.java
+++ /dev/null
@@ -1,354 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.trees;
-
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.net.URL;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Random;
-import java.util.Scanner;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import java.util.zip.GZIPInputStream;
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.examples.ExampleNodeStartup;
-import org.apache.ignite.examples.ml.MLExamplesCommonArgs;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.estimators.Estimators;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteTriFunction;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.trainers.columnbased.BiIndex;
-import org.apache.ignite.ml.trees.trainers.columnbased.BiIndexedCacheColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.GiniSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-import org.apache.ignite.ml.util.MnistUtils;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * <p>
- * Example of usage of decision trees algorithm for MNIST dataset
- * (it can be found here: http://yann.lecun.com/exdb/mnist/). </p>
- * <p>
- * Remote nodes should always be started with special configuration file which
- * enables P2P class loading: {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
- * <p>
- * Alternatively you can run {@link ExampleNodeStartup} in another JVM which will start node
- * with {@code examples/config/example-ignite.xml} configuration.</p>
- * <p>
- * It is recommended to start at least one node prior to launching this example if you intend
- * to run it with default memory settings.</p>
- * <p>
- * This example should be run with program arguments, for example
- * -cfg examples/config/example-ignite.xml.</p>
- * <p>
- * -cfg specifies path to a config path.</p>
- */
-public class DecisionTreesExample {
- /** Name of parameter specifying path of Ignite config. */
- private static final String CONFIG = "cfg";
-
- /** Default config path. */
- private static final String DEFAULT_CONFIG = "examples/config/example-ignite.xml";
-
- /**
- * Folder in which MNIST dataset is expected.
- */
- private static String MNIST_DIR = "examples/src/main/resources/";
-
- /**
- * Key for MNIST training images.
- */
- private static String MNIST_TRAIN_IMAGES = "train_images";
-
- /**
- * Key for MNIST training labels.
- */
- private static String MNIST_TRAIN_LABELS = "train_labels";
-
- /**
- * Key for MNIST test images.
- */
- private static String MNIST_TEST_IMAGES = "test_images";
-
- /**
- * Key for MNIST test labels.
- */
- private static String MNIST_TEST_LABELS = "test_labels";
-
- /**
- * Launches example.
- *
- * @param args Program arguments.
- */
- public static void main(String[] args) throws IOException {
- System.out.println(">>> Decision trees example started.");
-
- String igniteCfgPath;
-
- CommandLineParser parser = new BasicParser();
-
- String trainingImagesPath;
- String trainingLabelsPath;
-
- String testImagesPath;
- String testLabelsPath;
-
- Map<String, String> mnistPaths = new HashMap<>();
-
- mnistPaths.put(MNIST_TRAIN_IMAGES, "train-images-idx3-ubyte");
- mnistPaths.put(MNIST_TRAIN_LABELS, "train-labels-idx1-ubyte");
- mnistPaths.put(MNIST_TEST_IMAGES, "t10k-images-idx3-ubyte");
- mnistPaths.put(MNIST_TEST_LABELS, "t10k-labels-idx1-ubyte");
-
- try {
- // Parse the command line arguments.
- CommandLine line = parser.parse(buildOptions(), args);
-
- if (line.hasOption(MLExamplesCommonArgs.UNATTENDED)) {
- System.out.println(">>> Skipped example execution because 'unattended' mode is used.");
- System.out.println(">>> Decision trees example finished.");
- return;
- }
-
- igniteCfgPath = line.getOptionValue(CONFIG, DEFAULT_CONFIG);
- }
- catch (ParseException e) {
- e.printStackTrace();
- return;
- }
-
- if (!getMNIST(mnistPaths.values())) {
- System.out.println(">>> You should have MNIST dataset in " + MNIST_DIR + " to run this example.");
- return;
- }
-
- trainingImagesPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" +
- mnistPaths.get(MNIST_TRAIN_IMAGES))).getPath();
- trainingLabelsPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" +
- mnistPaths.get(MNIST_TRAIN_LABELS))).getPath();
- testImagesPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" +
- mnistPaths.get(MNIST_TEST_IMAGES))).getPath();
- testLabelsPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" +
- mnistPaths.get(MNIST_TEST_LABELS))).getPath();
-
- try (Ignite ignite = Ignition.start(igniteCfgPath)) {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
- int ptsCnt = 60000;
- int featCnt = 28 * 28;
-
- Stream<DenseLocalOnHeapVector> trainingMnistStream = MnistUtils.mnistAsStream(trainingImagesPath, trainingLabelsPath,
- new Random(123L), ptsCnt);
-
- Stream<DenseLocalOnHeapVector> testMnistStream = MnistUtils.mnistAsStream(testImagesPath, testLabelsPath,
- new Random(123L), 10_000);
-
- IgniteCache<BiIndex, Double> cache = createBiIndexedCache(ignite);
-
- loadVectorsIntoBiIndexedCache(cache.getName(), trainingMnistStream.iterator(), featCnt + 1, ignite);
-
- ColumnDecisionTreeTrainer<GiniSplitCalculator.GiniData> trainer = new ColumnDecisionTreeTrainer<>(10,
- ContinuousSplitCalculators.GINI.apply(ignite),
- RegionCalculators.GINI,
- RegionCalculators.MOST_COMMON,
- ignite);
-
- System.out.println(">>> Training started");
- long before = System.currentTimeMillis();
- DecisionTreeModel mdl = trainer.train(new BiIndexedCacheColumnDecisionTreeTrainerInput(cache, new HashMap<>(), ptsCnt, featCnt));
- System.out.println(">>> Training finished in " + (System.currentTimeMillis() - before));
-
- IgniteTriFunction<Model<Vector, Double>, Stream<IgniteBiTuple<Vector, Double>>, Function<Double, Double>, Double> mse =
- Estimators.errorsPercentage();
-
- Double accuracy = mse.apply(mdl, testMnistStream.map(v ->
- new IgniteBiTuple<>(v.viewPart(0, featCnt), v.getX(featCnt))), Function.identity());
-
- System.out.println(">>> Errs percentage: " + accuracy);
- }
- catch (IOException e) {
- e.printStackTrace();
- }
-
- System.out.println(">>> Decision trees example finished.");
- }
-
- /**
- * Get MNIST dataset. Value of predicate 'MNIST dataset is present in expected folder' is returned.
- *
- * @param mnistFileNames File names of MNIST dataset.
- * @return Value of predicate 'MNIST dataset is present in expected folder'.
- * @throws IOException In case of file system errors.
- */
- private static boolean getMNIST(Collection<String> mnistFileNames) throws IOException {
- List<String> missing = mnistFileNames.stream().
- filter(f -> IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" + f) == null).
- collect(Collectors.toList());
-
- if (!missing.isEmpty()) {
- System.out.println(">>> You have not fully downloaded MNIST dataset in directory " + MNIST_DIR +
- ", do you want it to be downloaded? [y]/n");
- Scanner s = new Scanner(System.in);
- String str = s.nextLine();
-
- if (!str.isEmpty() && !str.toLowerCase().equals("y"))
- return false;
- }
-
- for (String s : missing) {
- String f = s + ".gz";
- System.out.println(">>> Downloading " + f + "...");
- URL website = new URL("http://yann.lecun.com/exdb/mnistAsStream/" + f);
- ReadableByteChannel rbc = Channels.newChannel(website.openStream());
- FileOutputStream fos = new FileOutputStream(MNIST_DIR + "/" + f);
- fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
- System.out.println(">>> Done.");
-
- System.out.println(">>> Unzipping " + f + "...");
- unzip(MNIST_DIR + "/" + f, MNIST_DIR + "/" + s);
-
- System.out.println(">>> Deleting gzip " + f + ", status: " +
- Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" + f)).delete());
-
- System.out.println(">>> Done.");
- }
-
- return true;
- }
-
- /**
- * Unzip file located in {@code input} to {@code output}.
- *
- * @param input Input file path.
- * @param output Output file path.
- * @throws IOException In case of file system errors.
- */
- private static void unzip(String input, String output) throws IOException {
- byte[] buf = new byte[1024];
-
- try (GZIPInputStream gis = new GZIPInputStream(new FileInputStream(input));
- FileOutputStream out = new FileOutputStream(output)) {
- int sz;
- while ((sz = gis.read(buf)) > 0)
- out.write(buf, 0, sz);
- }
- }
-
- /**
- * Build cli options.
- */
- @NotNull private static Options buildOptions() {
- Options options = new Options();
-
- Option cfgOpt = OptionBuilder
- .withArgName(CONFIG)
- .withLongOpt(CONFIG)
- .hasArg()
- .withDescription("Path to the config.")
- .isRequired(false).create();
-
- Option unattended = OptionBuilder
- .withArgName(MLExamplesCommonArgs.UNATTENDED)
- .withLongOpt(MLExamplesCommonArgs.UNATTENDED)
- .withDescription("Is example run unattended.")
- .isRequired(false).create();
-
- options.addOption(cfgOpt);
- options.addOption(unattended);
-
- return options;
- }
-
- /**
- * Creates cache where data for training is stored.
- *
- * @param ignite Ignite instance.
- * @return cache where data for training is stored.
- */
- private static IgniteCache<BiIndex, Double> createBiIndexedCache(Ignite ignite) {
- CacheConfiguration<BiIndex, Double> cfg = new CacheConfiguration<>();
-
- // Write to primary.
- cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
- // No copying of values.
- cfg.setCopyOnRead(false);
-
- cfg.setName("TMP_BI_INDEXED_CACHE");
-
- return ignite.getOrCreateCache(cfg);
- }
-
- /**
- * Loads vectors into cache.
- *
- * @param cacheName Name of cache.
- * @param vectorsIter Iterator over vectors to load.
- * @param vectorSize Size of vector.
- * @param ignite Ignite instance.
- */
- private static void loadVectorsIntoBiIndexedCache(String cacheName, Iterator<? extends Vector> vectorsIter,
- int vectorSize, Ignite ignite) {
- try (IgniteDataStreamer<BiIndex, Double> streamer =
- ignite.dataStreamer(cacheName)) {
- int sampleIdx = 0;
-
- streamer.perNodeBufferSize(10000);
-
- while (vectorsIter.hasNext()) {
- org.apache.ignite.ml.math.Vector next = vectorsIter.next();
-
- for (int i = 0; i < vectorSize; i++)
- streamer.addData(new BiIndex(sampleIdx, i), next.getX(i));
-
- sampleIdx++;
-
- if (sampleIdx % 1000 == 0)
- System.out.println(">>> Loaded " + sampleIdx + " vectors.");
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/examples/src/main/java/org/apache/ignite/examples/ml/trees/package-info.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/trees/package-info.java b/examples/src/main/java/org/apache/ignite/examples/ml/trees/package-info.java
deleted file mode 100644
index d944f60..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/trees/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Decision trees examples.
- */
-package org.apache.ignite.examples.ml.trees;
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java
index 4e0a570..f53b801 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java
@@ -17,11 +17,8 @@
package org.apache.ignite.ml;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-
/**
* Interface for Trainers. Trainer is just a function which produces model from the data.
- * See for example {@link ColumnDecisionTreeTrainer}.
*
* @param <M> Type of produced model.
* @param <T> Type of data needed for model producing.
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
new file mode 100644
index 0000000..c0b88fc
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.primitive.builder.context.EmptyContextBuilder;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.trainers.DatasetTrainer;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.data.DecisionTreeDataBuilder;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+import org.apache.ignite.ml.tree.impurity.util.StepFunctionCompressor;
+import org.apache.ignite.ml.tree.leaf.DecisionTreeLeafBuilder;
+
+/**
+ * Distributed decision tree trainer that allows to fit trees using row-partitioned dataset.
+ *
+ * @param <T> Type of impurity measure.
+ */
+abstract class DecisionTree<T extends ImpurityMeasure<T>> implements DatasetTrainer<DecisionTreeNode, Double> {
+ /** Max tree deep. */
+ private final int maxDeep;
+
+ /** Min impurity decrease. */
+ private final double minImpurityDecrease;
+
+ /** Step function compressor. */
+ private final StepFunctionCompressor<T> compressor;
+
+ /** Decision tree leaf builder. */
+ private final DecisionTreeLeafBuilder decisionTreeLeafBuilder;
+
+ /**
+ * Constructs a new distributed decision tree trainer.
+ *
+ * @param maxDeep Max tree deep.
+ * @param minImpurityDecrease Min impurity decrease.
+ * @param compressor Impurity function compressor.
+ * @param decisionTreeLeafBuilder Decision tree leaf builder.
+ */
+ DecisionTree(int maxDeep, double minImpurityDecrease, StepFunctionCompressor<T> compressor, DecisionTreeLeafBuilder decisionTreeLeafBuilder) {
+ this.maxDeep = maxDeep;
+ this.minImpurityDecrease = minImpurityDecrease;
+ this.compressor = compressor;
+ this.decisionTreeLeafBuilder = decisionTreeLeafBuilder;
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K, V> DecisionTreeNode fit(DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+ try (Dataset<EmptyContext, DecisionTreeData> dataset = datasetBuilder.build(
+ new EmptyContextBuilder<>(),
+ new DecisionTreeDataBuilder<>(featureExtractor, lbExtractor)
+ )) {
+ return split(dataset, e -> true, 0, getImpurityMeasureCalculator(dataset));
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Returns impurity measure calculator.
+ *
+ * @param dataset Dataset.
+ * @return Impurity measure calculator.
+ */
+ abstract ImpurityMeasureCalculator<T> getImpurityMeasureCalculator(Dataset<EmptyContext, DecisionTreeData> dataset);
+
+ /**
+ * Splits the node specified by the given dataset and predicate and returns decision tree node.
+ *
+ * @param dataset Dataset.
+ * @param filter Decision tree node predicate.
+ * @param deep Current tree deep.
+ * @param impurityCalc Impurity measure calculator.
+ * @return Decision tree node.
+ */
+ private DecisionTreeNode split(Dataset<EmptyContext, DecisionTreeData> dataset, TreeFilter filter, int deep,
+ ImpurityMeasureCalculator<T> impurityCalc) {
+ if (deep >= maxDeep)
+ return decisionTreeLeafBuilder.createLeafNode(dataset, filter);
+
+ StepFunction<T>[] criterionFunctions = calculateImpurityForAllColumns(dataset, filter, impurityCalc);
+
+ if (criterionFunctions == null)
+ return decisionTreeLeafBuilder.createLeafNode(dataset, filter);
+
+ SplitPoint splitPnt = calculateBestSplitPoint(criterionFunctions);
+
+ if (splitPnt == null)
+ return decisionTreeLeafBuilder.createLeafNode(dataset, filter);
+
+ return new DecisionTreeConditionalNode(
+ splitPnt.col,
+ splitPnt.threshold,
+ split(dataset, updatePredicateForThenNode(filter, splitPnt), deep + 1, impurityCalc),
+ split(dataset, updatePredicateForElseNode(filter, splitPnt), deep + 1, impurityCalc)
+ );
+ }
+
+ /**
+ * Calculates impurity measure functions for all columns for the node specified by the given dataset and predicate.
+ *
+ * @param dataset Dataset.
+ * @param filter Decision tree node predicate.
+ * @param impurityCalc Impurity measure calculator.
+ * @return Array of impurity measure functions for all columns.
+ */
+ private StepFunction<T>[] calculateImpurityForAllColumns(Dataset<EmptyContext, DecisionTreeData> dataset,
+ TreeFilter filter, ImpurityMeasureCalculator<T> impurityCalc) {
+ return dataset.compute(
+ part -> {
+ if (compressor != null)
+ return compressor.compress(impurityCalc.calculate(part.filter(filter)));
+ else
+ return impurityCalc.calculate(part.filter(filter));
+ }, this::reduce
+ );
+ }
+
+ /**
+ * Calculates best split point.
+ *
+ * @param criterionFunctions Array of impurity measure functions for all columns.
+ * @return Best split point.
+ */
+ private SplitPoint calculateBestSplitPoint(StepFunction<T>[] criterionFunctions) {
+ SplitPoint<T> res = null;
+
+ for (int col = 0; col < criterionFunctions.length; col++) {
+ StepFunction<T> criterionFunctionForCol = criterionFunctions[col];
+
+ double[] arguments = criterionFunctionForCol.getX();
+ T[] values = criterionFunctionForCol.getY();
+
+ for (int leftSize = 1; leftSize < values.length - 1; leftSize++) {
+ if ((values[0].impurity() - values[leftSize].impurity()) > minImpurityDecrease
+ && (res == null || values[leftSize].compareTo(res.val) < 0))
+ res = new SplitPoint<>(values[leftSize], col, calculateThreshold(arguments, leftSize));
+ }
+ }
+
+ return res;
+ }
+
+ /**
+ * Merges two arrays gotten from two partitions.
+ *
+ * @param a First step function.
+ * @param b Second step function.
+ * @return Merged step function.
+ */
+ private StepFunction<T>[] reduce(StepFunction<T>[] a, StepFunction<T>[] b) {
+ if (a == null)
+ return b;
+ if (b == null)
+ return a;
+ else {
+ StepFunction<T>[] res = Arrays.copyOf(a, a.length);
+
+ for (int i = 0; i < res.length; i++)
+ res[i] = res[i].add(b[i]);
+
+ return res;
+ }
+ }
+
+ /**
+ * Calculates threshold based on the given step function arguments and split point (specified left size).
+ *
+ * @param arguments Step function arguments.
+ * @param leftSize Split point (left size).
+ * @return Threshold.
+ */
+ private double calculateThreshold(double[] arguments, int leftSize) {
+ return (arguments[leftSize] + arguments[leftSize + 1]) / 2.0;
+ }
+
+ /**
+ * Constructs a new predicate for "then" node based on the parent node predicate and split point.
+ *
+ * @param filter Parent node predicate.
+ * @param splitPnt Split point.
+ * @return Predicate for "then" node.
+ */
+ private TreeFilter updatePredicateForThenNode(TreeFilter filter, SplitPoint splitPnt) {
+ return filter.and(f -> f[splitPnt.col] > splitPnt.threshold);
+ }
+
+ /**
+ * Constructs a new predicate for "else" node based on the parent node predicate and split point.
+ *
+ * @param filter Parent node predicate.
+ * @param splitPnt Split point.
+ * @return Predicate for "else" node.
+ */
+ private TreeFilter updatePredicateForElseNode(TreeFilter filter, SplitPoint splitPnt) {
+ return filter.and(f -> f[splitPnt.col] <= splitPnt.threshold);
+ }
+
+ /**
+ * Util class that represents split point.
+ */
+ private static class SplitPoint<T extends ImpurityMeasure<T>> implements Serializable {
+ /** */
+ private static final long serialVersionUID = -1758525953544425043L;
+
+ /** Split point impurity measure value. */
+ private final T val;
+
+ /** Column. */
+ private final int col;
+
+ /** Threshold. */
+ private final double threshold;
+
+ /**
+ * Constructs a new instance of split point.
+ *
+ * @param val Split point impurity measure value.
+ * @param col Column.
+ * @param threshold Threshold.
+ */
+ SplitPoint(T val, int col, double threshold) {
+ this.val = val;
+ this.col = col;
+ this.threshold = threshold;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java
new file mode 100644
index 0000000..ce75190
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.gini.GiniImpurityMeasure;
+import org.apache.ignite.ml.tree.impurity.gini.GiniImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunctionCompressor;
+import org.apache.ignite.ml.tree.leaf.MostCommonDecisionTreeLeafBuilder;
+
+/**
+ * Decision tree classifier based on distributed decision tree trainer that allows to fit trees using row-partitioned
+ * dataset.
+ */
+public class DecisionTreeClassificationTrainer extends DecisionTree<GiniImpurityMeasure> {
+ /**
+ * Constructs a new decision tree classifier with default impurity function compressor.
+ *
+ * @param maxDeep Max tree deep.
+ * @param minImpurityDecrease Min impurity decrease.
+ */
+ public DecisionTreeClassificationTrainer(int maxDeep, double minImpurityDecrease) {
+ this(maxDeep, minImpurityDecrease, null);
+ }
+
+ /**
+ * Constructs a new instance of decision tree classifier.
+ *
+ * @param maxDeep Max tree deep.
+ * @param minImpurityDecrease Min impurity decrease.
+ */
+ public DecisionTreeClassificationTrainer(int maxDeep, double minImpurityDecrease,
+ StepFunctionCompressor<GiniImpurityMeasure> compressor) {
+ super(maxDeep, minImpurityDecrease, compressor, new MostCommonDecisionTreeLeafBuilder());
+ }
+
+ /** {@inheritDoc} */
+ @Override ImpurityMeasureCalculator<GiniImpurityMeasure> getImpurityMeasureCalculator(
+ Dataset<EmptyContext, DecisionTreeData> dataset) {
+ Set<Double> labels = dataset.compute(part -> {
+
+ if (part.getLabels() != null) {
+ Set<Double> list = new HashSet<>();
+
+ for (double lb : part.getLabels())
+ list.add(lb);
+
+ return list;
+ }
+
+ return null;
+ }, (a, b) -> {
+ if (a == null)
+ return b;
+ else if (b == null)
+ return a;
+ else {
+ a.addAll(b);
+ return a;
+ }
+ });
+
+ Map<Double, Integer> encoder = new HashMap<>();
+
+ int idx = 0;
+ for (Double lb : labels)
+ encoder.put(lb, idx++);
+
+ return new GiniImpurityMeasureCalculator(encoder);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java
new file mode 100644
index 0000000..9818239
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+/**
+ * Decision tree conditional (non-leaf) node.
+ */
+public class DecisionTreeConditionalNode implements DecisionTreeNode {
+ /** */
+ private static final long serialVersionUID = 981630737007982172L;
+
+ /** Column of the value to be tested. */
+ private final int col;
+
+ /** Threshold. */
+ private final double threshold;
+
+ /** Node that will be used in case tested value is greater then threshold. */
+ private final DecisionTreeNode thenNode;
+
+ /** Node that will be used in case tested value is not greater then threshold. */
+ private final DecisionTreeNode elseNode;
+
+ /**
+ * Constructs a new instance of decision tree conditional node.
+ *
+ * @param col Column of the value to be tested.
+ * @param threshold Threshold.
+ * @param thenNode Node that will be used in case tested value is greater then threshold.
+ * @param elseNode Node that will be used in case tested value is not greater then threshold.
+ */
+ DecisionTreeConditionalNode(int col, double threshold, DecisionTreeNode thenNode, DecisionTreeNode elseNode) {
+ this.col = col;
+ this.threshold = threshold;
+ this.thenNode = thenNode;
+ this.elseNode = elseNode;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Double apply(double[] features) {
+ return features[col] > threshold ? thenNode.apply(features) : elseNode.apply(features);
+ }
+
+ /** */
+ public int getCol() {
+ return col;
+ }
+
+ /** */
+ public double getThreshold() {
+ return threshold;
+ }
+
+ /** */
+ public DecisionTreeNode getThenNode() {
+ return thenNode;
+ }
+
+ /** */
+ public DecisionTreeNode getElseNode() {
+ return elseNode;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java
new file mode 100644
index 0000000..4c6369d
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+/**
+ * Decision tree leaf node which contains value.
+ */
+public class DecisionTreeLeafNode implements DecisionTreeNode {
+ /** */
+ private static final long serialVersionUID = -472145568088482206L;
+
+ /** Value of the node. */
+ private final double val;
+
+ /**
+ * Constructs a new decision tree leaf node.
+ *
+ * @param val Value of the node.
+ */
+ public DecisionTreeLeafNode(double val) {
+ this.val = val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Double apply(double[] doubles) {
+ return val;
+ }
+
+ /** */
+ public double getVal() {
+ return val;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeNode.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeNode.java
new file mode 100644
index 0000000..94878eb
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeNode.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import org.apache.ignite.ml.Model;
+
+/**
+ * Base interface for decision tree nodes.
+ */
+public interface DecisionTreeNode extends Model<double[], Double> {
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java
new file mode 100644
index 0000000..2bf09d3
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.mse.MSEImpurityMeasure;
+import org.apache.ignite.ml.tree.impurity.mse.MSEImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunctionCompressor;
+import org.apache.ignite.ml.tree.leaf.MeanDecisionTreeLeafBuilder;
+
+/**
+ * Decision tree regressor based on distributed decision tree trainer that allows to fit trees using row-partitioned
+ * dataset.
+ */
+public class DecisionTreeRegressionTrainer extends DecisionTree<MSEImpurityMeasure> {
+ /**
+ * Constructs a new decision tree regressor with default impurity function compressor.
+ *
+ * @param maxDeep Max tree deep.
+ * @param minImpurityDecrease Min impurity decrease.
+ */
+ public DecisionTreeRegressionTrainer(int maxDeep, double minImpurityDecrease) {
+ this(maxDeep, minImpurityDecrease, null);
+ }
+
+ /**
+ * Constructs a new decision tree regressor.
+ *
+ * @param maxDeep Max tree deep.
+ * @param minImpurityDecrease Min impurity decrease.
+ */
+ public DecisionTreeRegressionTrainer(int maxDeep, double minImpurityDecrease,
+ StepFunctionCompressor<MSEImpurityMeasure> compressor) {
+ super(maxDeep, minImpurityDecrease, compressor, new MeanDecisionTreeLeafBuilder());
+ }
+
+ /** {@inheritDoc} */
+ @Override ImpurityMeasureCalculator<MSEImpurityMeasure> getImpurityMeasureCalculator(
+ Dataset<EmptyContext, DecisionTreeData> dataset) {
+ return new MSEImpurityMeasureCalculator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/TreeFilter.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/TreeFilter.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/TreeFilter.java
new file mode 100644
index 0000000..3e4dc00
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/TreeFilter.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.io.Serializable;
+import java.util.Objects;
+import java.util.function.Predicate;
+
+/**
+ * Predicate used to define objects that placed in decision tree node.
+ */
+public interface TreeFilter extends Predicate<double[]>, Serializable {
+ /**
+ * Returns a composed predicate.
+ *
+ * @param other Predicate that will be logically-ANDed with this predicate.
+ * @return Returns a composed predicate
+ */
+ default TreeFilter and(TreeFilter other) {
+ Objects.requireNonNull(other);
+ return (t) -> test(t) && other.test(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java
new file mode 100644
index 0000000..34deb46
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.data;
+
+import org.apache.ignite.ml.tree.TreeFilter;
+
+/**
+ * A partition {@code data} of the containing matrix of features and vector of labels stored in heap.
+ */
+public class DecisionTreeData implements AutoCloseable {
+ /** Matrix with features. */
+ private final double[][] features;
+
+ /** Vector with labels. */
+ private final double[] labels;
+
+ /**
+ * Constructs a new instance of decision tree data.
+ *
+ * @param features Matrix with features.
+ * @param labels Vector with labels.
+ */
+ public DecisionTreeData(double[][] features, double[] labels) {
+ assert features.length == labels.length : "Features and labels have to be the same length";
+
+ this.features = features;
+ this.labels = labels;
+ }
+
+ /**
+ * Filters objects and returns only data that passed filter.
+ *
+ * @param filter Filter.
+ * @return Data passed filter.
+ */
+ public DecisionTreeData filter(TreeFilter filter) {
+ int size = 0;
+
+ for (int i = 0; i < features.length; i++)
+ if (filter.test(features[i]))
+ size++;
+
+ double[][] newFeatures = new double[size][];
+ double[] newLabels = new double[size];
+
+ int ptr = 0;
+
+ for (int i = 0; i < features.length; i++) {
+ if (filter.test(features[i])) {
+ newFeatures[ptr] = features[i];
+ newLabels[ptr] = labels[i];
+
+ ptr++;
+ }
+ }
+
+ return new DecisionTreeData(newFeatures, newLabels);
+ }
+
+ /**
+ * Sorts data by specified column in ascending order.
+ *
+ * @param col Column.
+ */
+ public void sort(int col) {
+ sort(col, 0, features.length - 1);
+ }
+
+ /** */
+ private void sort(int col, int from, int to) {
+ if (from < to) {
+ double pivot = features[(from + to) / 2][col];
+
+ int i = from, j = to;
+
+ while (i <= j) {
+ while (features[i][col] < pivot) i++;
+ while (features[j][col] > pivot) j--;
+
+ if (i <= j) {
+ double[] tmpFeature = features[i];
+ features[i] = features[j];
+ features[j] = tmpFeature;
+
+ double tmpLb = labels[i];
+ labels[i] = labels[j];
+ labels[j] = tmpLb;
+
+ i++;
+ j--;
+ }
+ }
+
+ sort(col, from, j);
+ sort(col, i, to);
+ }
+ }
+
+ /** */
+ public double[][] getFeatures() {
+ return features;
+ }
+
+ /** */
+ public double[] getLabels() {
+ return labels;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() {
+ // Do nothing, GC will clean up.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeDataBuilder.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeDataBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeDataBuilder.java
new file mode 100644
index 0000000..67109ae
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeDataBuilder.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.data;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import org.apache.ignite.ml.dataset.PartitionDataBuilder;
+import org.apache.ignite.ml.dataset.UpstreamEntry;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+
+/**
+ * A partition {@code data} builder that makes {@link DecisionTreeData}.
+ *
+ * @param <K> Type of a key in <tt>upstream</tt> data.
+ * @param <V> Type of a value in <tt>upstream</tt> data.
+ * @param <C> Type of a partition <tt>context</tt>.
+ */
+public class DecisionTreeDataBuilder<K, V, C extends Serializable>
+ implements PartitionDataBuilder<K, V, C, DecisionTreeData> {
+ /** */
+ private static final long serialVersionUID = 3678784980215216039L;
+
+ /** Function that extracts features from an {@code upstream} data. */
+ private final IgniteBiFunction<K, V, double[]> featureExtractor;
+
+ /** Function that extracts labels from an {@code upstream} data. */
+ private final IgniteBiFunction<K, V, Double> lbExtractor;
+
+ /**
+ * Constructs a new instance of decision tree data builder.
+ *
+ * @param featureExtractor Function that extracts features from an {@code upstream} data.
+ * @param lbExtractor Function that extracts labels from an {@code upstream} data.
+ */
+ public DecisionTreeDataBuilder(IgniteBiFunction<K, V, double[]> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
+ this.featureExtractor = featureExtractor;
+ this.lbExtractor = lbExtractor;
+ }
+
+ /** {@inheritDoc} */
+ @Override public DecisionTreeData build(Iterator<UpstreamEntry<K, V>> upstreamData, long upstreamDataSize, C ctx) {
+ double[][] features = new double[Math.toIntExact(upstreamDataSize)][];
+ double[] labels = new double[Math.toIntExact(upstreamDataSize)];
+
+ int ptr = 0;
+ while (upstreamData.hasNext()) {
+ UpstreamEntry<K, V> entry = upstreamData.next();
+
+ features[ptr] = featureExtractor.apply(entry.getKey(), entry.getValue());
+ labels[ptr] = lbExtractor.apply(entry.getKey(), entry.getValue());
+
+ ptr++;
+ }
+
+ return new DecisionTreeData(features, labels);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/package-info.java
new file mode 100644
index 0000000..192b07f
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains data and data builder required for decision tree trainers built on top of partition based dataset.
+ */
+package org.apache.ignite.ml.tree.data;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasure.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasure.java
new file mode 100644
index 0000000..7ad2b80
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasure.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity;
+
+import java.io.Serializable;
+
+/**
+ * Base interface for impurity measures that can be used in distributed decision tree algorithm.
+ *
+ * @param <T> Type of this impurity measure.
+ */
+public interface ImpurityMeasure<T extends ImpurityMeasure<T>> extends Comparable<T>, Serializable {
+ /**
+ * Calculates impurity measure as a single double value.
+ *
+ * @return Impurity measure value.
+ */
+ public double impurity();
+
+ /**
+ * Adds the given impurity to this.
+ *
+ * @param measure Another impurity.
+ * @return Sum of this and the given impurity.
+ */
+ public T add(T measure);
+
+ /**
+ * Subtracts the given impurity for this.
+ *
+ * @param measure Another impurity.
+ * @return Difference of this and the given impurity.
+ */
+ public T subtract(T measure);
+
+ /** {@inheritDoc} */
+ default public int compareTo(T o) {
+ return Double.compare(impurity(), o.impurity());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasureCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasureCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasureCalculator.java
new file mode 100644
index 0000000..2b69356
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasureCalculator.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity;
+
+import java.io.Serializable;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+
+/**
+ * Base interface for impurity measure calculators that calculates all impurity measures required to find a best split.
+ *
+ * @param <T> Type of impurity measure.
+ */
+public interface ImpurityMeasureCalculator<T extends ImpurityMeasure<T>> extends Serializable {
+ /**
+ * Calculates all impurity measures required required to find a best split and returns them as an array of
+ * {@link StepFunction} (for every column).
+ *
+ * @param data Features and labels.
+ * @return Impurity measures as an array of {@link StepFunction} (for every column).
+ */
+ public StepFunction<T>[] calculate(DecisionTreeData data);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasure.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasure.java
new file mode 100644
index 0000000..817baf5
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasure.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.gini;
+
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Gini impurity measure which is calculated the following way:
+ * {@code \-frac{1}{L}\sum_{i=1}^{s}l_i^2 - \frac{1}{R}\sum_{i=s+1}^{n}r_i^2}.
+ */
+public class GiniImpurityMeasure implements ImpurityMeasure<GiniImpurityMeasure> {
+ /** */
+ private static final long serialVersionUID = 5338129703395229970L;
+
+ /** Number of elements of each type in the left part. */
+ private final long[] left;
+
+ /** Number of elements of each type in the right part. */
+ private final long[] right;
+
+ /**
+ * Constructs a new instance of Gini impurity measure.
+ *
+ * @param left Number of elements of each type in the left part.
+ * @param right Number of elements of each type in the right part.
+ */
+ GiniImpurityMeasure(long[] left, long[] right) {
+ assert left.length == right.length : "Left and right parts have to be the same length";
+
+ this.left = left;
+ this.right = right;
+ }
+
+ /** {@inheritDoc} */
+ @Override public double impurity() {
+ long leftCnt = 0;
+ long rightCnt = 0;
+
+ double leftImpurity = 0;
+ double rightImpurity = 0;
+
+ for (long e : left)
+ leftCnt += e;
+
+ for (long e : right)
+ rightCnt += e;
+
+ if (leftCnt > 0)
+ for (long e : left)
+ leftImpurity += Math.pow(e, 2) / leftCnt;
+
+ if (rightCnt > 0)
+ for (long e : right)
+ rightImpurity += Math.pow(e, 2) / rightCnt;
+
+ return -(leftImpurity + rightImpurity);
+ }
+
+ /** {@inheritDoc} */
+ @Override public GiniImpurityMeasure add(GiniImpurityMeasure b) {
+ assert left.length == b.left.length : "Subtracted measure has to have length " + left.length;
+ assert left.length == b.right.length : "Subtracted measure has to have length " + left.length;
+
+ long[] leftRes = new long[left.length];
+ long[] rightRes = new long[left.length];
+
+ for (int i = 0; i < left.length; i++) {
+ leftRes[i] = left[i] + b.left[i];
+ rightRes[i] = right[i] + b.right[i];
+ }
+
+ return new GiniImpurityMeasure(leftRes, rightRes);
+ }
+
+ /** {@inheritDoc} */
+ @Override public GiniImpurityMeasure subtract(GiniImpurityMeasure b) {
+ assert left.length == b.left.length : "Subtracted measure has to have length " + left.length;
+ assert left.length == b.right.length : "Subtracted measure has to have length " + left.length;
+
+ long[] leftRes = new long[left.length];
+ long[] rightRes = new long[left.length];
+
+ for (int i = 0; i < left.length; i++) {
+ leftRes[i] = left[i] - b.left[i];
+ rightRes[i] = right[i] - b.right[i];
+ }
+
+ return new GiniImpurityMeasure(leftRes, rightRes);
+ }
+
+ /** */
+ public long[] getLeft() {
+ return left;
+ }
+
+ /** */
+ public long[] getRight() {
+ return right;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculator.java
new file mode 100644
index 0000000..0dd0a10
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculator.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.gini;
+
+import java.util.Arrays;
+import java.util.Map;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+
+/**
+ * Gini impurity measure calculator.
+ */
+public class GiniImpurityMeasureCalculator implements ImpurityMeasureCalculator<GiniImpurityMeasure> {
+ /** */
+ private static final long serialVersionUID = -522995134128519679L;
+
+ /** Label encoder which defines integer value for every label class. */
+ private final Map<Double, Integer> lbEncoder;
+
+ /**
+ * Constructs a new instance of Gini impurity measure calculator.
+ *
+ * @param lbEncoder Label encoder which defines integer value for every label class.
+ */
+ public GiniImpurityMeasureCalculator(Map<Double, Integer> lbEncoder) {
+ this.lbEncoder = lbEncoder;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public StepFunction<GiniImpurityMeasure>[] calculate(DecisionTreeData data) {
+ double[][] features = data.getFeatures();
+ double[] labels = data.getLabels();
+
+ if (features.length > 0) {
+ StepFunction<GiniImpurityMeasure>[] res = new StepFunction[features[0].length];
+
+ for (int col = 0; col < res.length; col++) {
+ data.sort(col);
+
+ double[] x = new double[features.length + 1];
+ GiniImpurityMeasure[] y = new GiniImpurityMeasure[features.length + 1];
+
+ int xPtr = 0, yPtr = 0;
+
+ long[] left = new long[lbEncoder.size()];
+ long[] right = new long[lbEncoder.size()];
+
+ for (int i = 0; i < labels.length; i++)
+ right[getLabelCode(labels[i])]++;
+
+ x[xPtr++] = Double.NEGATIVE_INFINITY;
+ y[yPtr++] = new GiniImpurityMeasure(
+ Arrays.copyOf(left, left.length),
+ Arrays.copyOf(right, right.length)
+ );
+
+ for (int i = 0; i < features.length; i++) {
+ left[getLabelCode(labels[i])]++;
+ right[getLabelCode(labels[i])]--;
+
+ if (i < (features.length - 1) && features[i + 1][col] == features[i][col])
+ continue;
+
+ x[xPtr++] = features[i][col];
+ y[yPtr++] = new GiniImpurityMeasure(
+ Arrays.copyOf(left, left.length),
+ Arrays.copyOf(right, right.length)
+ );
+ }
+
+ res[col] = new StepFunction<>(Arrays.copyOf(x, xPtr), Arrays.copyOf(y, yPtr));
+ }
+
+ return res;
+ }
+
+ return null;
+ }
+
+ /**
+ * Returns label code.
+ *
+ * @param lb Label.
+ * @return Label code.
+ */
+ int getLabelCode(double lb) {
+ Integer code = lbEncoder.get(lb);
+
+ assert code != null : "Can't find code for label " + lb;
+
+ return code;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/package-info.java
new file mode 100644
index 0000000..d14cd92
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains Gini impurity measure and calculator.
+ */
+package org.apache.ignite.ml.tree.impurity.gini;
\ No newline at end of file
[03/14] ignite git commit: IGNITE-8059: Integrate decision tree with
partition based dataset.
Posted by ak...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/VarianceSplitCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/VarianceSplitCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/VarianceSplitCalculator.java
deleted file mode 100644
index 66c54f2..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/VarianceSplitCalculator.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs;
-
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.PrimitiveIterator;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.ContinuousSplitInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-
-/**
- * Calculator of variance in a given region.
- */
-public class VarianceSplitCalculator implements ContinuousSplitCalculator<VarianceSplitCalculator.VarianceData> {
- /**
- * Data used in variance calculations.
- */
- public static class VarianceData extends ContinuousRegionInfo {
- /** Mean value in a given region. */
- double mean;
-
- /**
- * @param var Variance in this region.
- * @param size Size of data for which variance is calculated.
- * @param mean Mean value in this region.
- */
- public VarianceData(double var, int size, double mean) {
- super(var, size);
- this.mean = mean;
- }
-
- /**
- * No-op constructor. For serialization/deserialization.
- */
- public VarianceData() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- super.writeExternal(out);
- out.writeDouble(mean);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- super.readExternal(in);
- mean = in.readDouble();
- }
-
- /**
- * Returns mean.
- */
- public double mean() {
- return mean;
- }
- }
-
- /** {@inheritDoc} */
- @Override public VarianceData calculateRegionInfo(DoubleStream s, int size) {
- PrimitiveIterator.OfDouble itr = s.iterator();
- int i = 0;
-
- double mean = 0.0;
- double m2 = 0.0;
-
- // Here we calculate variance and mean by incremental computation.
- while (itr.hasNext()) {
- i++;
- double x = itr.next();
- double delta = x - mean;
- mean += delta / i;
- double delta2 = x - mean;
- m2 += delta * delta2;
- }
-
- return new VarianceData(m2 / i, size, mean);
- }
-
- /** {@inheritDoc} */
- @Override public SplitInfo<VarianceData> splitRegion(Integer[] s, double[] values, double[] labels, int regionIdx,
- VarianceData d) {
- int size = d.getSize();
-
- double lm2 = 0.0;
- double rm2 = d.impurity() * size;
- int lSize = size;
-
- double lMean = 0.0;
- double rMean = d.mean;
-
- double minImpurity = d.impurity() * size;
- double curThreshold;
- double curImpurity;
- double threshold = Double.NEGATIVE_INFINITY;
-
- int i = 0;
- int nextIdx = s[0];
- i++;
- double[] lrImps = new double[] {lm2, rm2, lMean, rMean};
-
- do {
- // Process all values equal to prev.
- while (i < s.length) {
- moveLeft(labels[nextIdx], lrImps[2], i, lrImps[0], lrImps[3], size - i, lrImps[1], lrImps);
- curImpurity = (lrImps[0] + lrImps[1]);
- curThreshold = values[nextIdx];
-
- if (values[nextIdx] != values[(nextIdx = s[i++])]) {
- if (curImpurity < minImpurity) {
- lSize = i - 1;
-
- lm2 = lrImps[0];
- rm2 = lrImps[1];
-
- lMean = lrImps[2];
- rMean = lrImps[3];
-
- minImpurity = curImpurity;
- threshold = curThreshold;
- }
-
- break;
- }
- }
- }
- while (i < s.length - 1);
-
- if (lSize == size)
- return null;
-
- VarianceData lData = new VarianceData(lm2 / (lSize != 0 ? lSize : 1), lSize, lMean);
- int rSize = size - lSize;
- VarianceData rData = new VarianceData(rm2 / (rSize != 0 ? rSize : 1), rSize, rMean);
-
- return new ContinuousSplitInfo<>(regionIdx, threshold, lData, rData);
- }
-
- /**
- * Add point to the left interval and remove it from the right interval and calculate necessary statistics on
- * intervals with new bounds.
- */
- private void moveLeft(double x, double lMean, int lSize, double lm2, double rMean, int rSize, double rm2,
- double[] data) {
- // We add point to the left interval.
- double lDelta = x - lMean;
- double lMeanNew = lMean + lDelta / lSize;
- double lm2New = lm2 + lDelta * (x - lMeanNew);
-
- // We remove point from the right interval. lSize + 1 is the size of right interval before removal.
- double rMeanNew = (rMean * (rSize + 1) - x) / rSize;
- double rm2New = rm2 - (x - rMean) * (x - rMeanNew);
-
- data[0] = lm2New;
- data[1] = rm2New;
-
- data[2] = lMeanNew;
- data[3] = rMeanNew;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/package-info.java
deleted file mode 100644
index 08c8a75..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Calculators of splits by continuous features.
- */
-package org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/package-info.java
deleted file mode 100644
index 8523914..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains column based decision tree algorithms.
- */
-package org.apache.ignite.ml.trees.trainers.columnbased;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/RegionCalculators.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/RegionCalculators.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/RegionCalculators.java
deleted file mode 100644
index 5c4b354..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/RegionCalculators.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.regcalcs;
-
-import it.unimi.dsi.fastutil.doubles.Double2IntOpenHashMap;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.PrimitiveIterator;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainerInput;
-
-/** Some commonly used functions for calculations of regions of space which correspond to decision tree leaf nodes. */
-public class RegionCalculators {
- /** Mean value in the region. */
- public static final IgniteFunction<DoubleStream, Double> MEAN = s -> s.average().orElse(0.0);
-
- /** Most common value in the region. */
- public static final IgniteFunction<DoubleStream, Double> MOST_COMMON =
- s -> {
- PrimitiveIterator.OfDouble itr = s.iterator();
- Map<Double, Integer> voc = new HashMap<>();
-
- while (itr.hasNext())
- voc.compute(itr.next(), (d, i) -> i != null ? i + 1 : 0);
-
- return voc.entrySet().stream().max(Comparator.comparing(Map.Entry::getValue)).map(Map.Entry::getKey).orElse(0.0);
- };
-
- /** Variance of a region. */
- public static final IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> VARIANCE = input ->
- s -> {
- PrimitiveIterator.OfDouble itr = s.iterator();
- int i = 0;
-
- double mean = 0.0;
- double m2 = 0.0;
-
- while (itr.hasNext()) {
- i++;
- double x = itr.next();
- double delta = x - mean;
- mean += delta / i;
- double delta2 = x - mean;
- m2 += delta * delta2;
- }
-
- return i > 0 ? m2 / i : 0.0;
- };
-
- /** Gini impurity of a region. */
- public static final IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> GINI = input ->
- s -> {
- PrimitiveIterator.OfDouble itr = s.iterator();
-
- Double2IntOpenHashMap m = new Double2IntOpenHashMap();
-
- int size = 0;
-
- while (itr.hasNext()) {
- size++;
- m.compute(itr.next(), (a, i) -> i != null ? i + 1 : 1);
- }
-
- double c2 = m.values().stream().mapToDouble(v -> v * v).sum();
-
- return size != 0 ? 1 - c2 / (size * size) : 0.0;
- };
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/package-info.java
deleted file mode 100644
index e8edd8f..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Region calculators.
- */
-package org.apache.ignite.ml.trees.trainers.columnbased.regcalcs;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/CategoricalFeatureProcessor.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/CategoricalFeatureProcessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/CategoricalFeatureProcessor.java
deleted file mode 100644
index 3232ac2..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/CategoricalFeatureProcessor.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.CategoricalRegionInfo;
-import org.apache.ignite.ml.trees.CategoricalSplitInfo;
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.RegionProjection;
-
-import static org.apache.ignite.ml.trees.trainers.columnbased.vectors.FeatureVectorProcessorUtils.splitByBitSet;
-
-/**
- * Categorical feature vector processor implementation used by {@link ColumnDecisionTreeTrainer}.
- */
-public class CategoricalFeatureProcessor
- implements FeatureProcessor<CategoricalRegionInfo, CategoricalSplitInfo<CategoricalRegionInfo>> {
- /** Count of categories for this feature. */
- private final int catsCnt;
-
- /** Function for calculating impurity of a given region of points. */
- private final IgniteFunction<DoubleStream, Double> calc;
-
- /**
- * @param calc Function for calculating impurity of a given region of points.
- * @param catsCnt Number of categories.
- */
- public CategoricalFeatureProcessor(IgniteFunction<DoubleStream, Double> calc, int catsCnt) {
- this.calc = calc;
- this.catsCnt = catsCnt;
- }
-
- /** */
- private SplitInfo<CategoricalRegionInfo> split(BitSet leftCats, int intervalIdx, Map<Integer, Integer> mapping,
- Integer[] sampleIndexes, double[] values, double[] labels, double impurity) {
- Map<Boolean, List<Integer>> leftRight = Arrays.stream(sampleIndexes).
- collect(Collectors.partitioningBy((smpl) -> leftCats.get(mapping.get((int)values[smpl]))));
-
- List<Integer> left = leftRight.get(true);
- int leftSize = left.size();
- double leftImpurity = calc.apply(left.stream().mapToDouble(s -> labels[s]));
-
- List<Integer> right = leftRight.get(false);
- int rightSize = right.size();
- double rightImpurity = calc.apply(right.stream().mapToDouble(s -> labels[s]));
-
- int totalSize = leftSize + rightSize;
-
- // Result of this call will be sent back to trainer node, we do not need vectors inside of sent data.
- CategoricalSplitInfo<CategoricalRegionInfo> res = new CategoricalSplitInfo<>(intervalIdx,
- new CategoricalRegionInfo(leftImpurity, null), // cats can be computed on the last step.
- new CategoricalRegionInfo(rightImpurity, null),
- leftCats);
-
- res.setInfoGain(impurity - (double)leftSize / totalSize * leftImpurity - (double)rightSize / totalSize * rightImpurity);
- return res;
- }
-
- /**
- * Get a stream of subsets given categories count.
- *
- * @param catsCnt categories count.
- * @return Stream of subsets given categories count.
- */
- private Stream<BitSet> powerSet(int catsCnt) {
- Iterable<BitSet> iterable = () -> new PSI(catsCnt);
- return StreamSupport.stream(iterable.spliterator(), false);
- }
-
- /** {@inheritDoc} */
- @Override public SplitInfo findBestSplit(RegionProjection<CategoricalRegionInfo> regionPrj, double[] values,
- double[] labels, int regIdx) {
- Map<Integer, Integer> mapping = mapping(regionPrj.data().cats());
-
- return powerSet(regionPrj.data().cats().length()).
- map(s -> split(s, regIdx, mapping, regionPrj.sampleIndexes(), values, labels, regionPrj.data().impurity())).
- max(Comparator.comparingDouble(SplitInfo::infoGain)).
- orElse(null);
- }
-
- /** {@inheritDoc} */
- @Override public RegionProjection<CategoricalRegionInfo> createInitialRegion(Integer[] sampleIndexes,
- double[] values, double[] labels) {
- BitSet set = new BitSet();
- set.set(0, catsCnt);
-
- Double impurity = calc.apply(Arrays.stream(labels));
-
- return new RegionProjection<>(sampleIndexes, new CategoricalRegionInfo(impurity, set), 0);
- }
-
- /** {@inheritDoc} */
- @Override public SparseBitSet calculateOwnershipBitSet(RegionProjection<CategoricalRegionInfo> regionPrj,
- double[] values,
- CategoricalSplitInfo<CategoricalRegionInfo> s) {
- SparseBitSet res = new SparseBitSet();
- Arrays.stream(regionPrj.sampleIndexes()).forEach(smpl -> res.set(smpl, s.bitSet().get((int)values[smpl])));
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public IgniteBiTuple<RegionProjection, RegionProjection> performSplit(SparseBitSet bs,
- RegionProjection<CategoricalRegionInfo> reg, CategoricalRegionInfo leftData, CategoricalRegionInfo rightData) {
- return performSplitGeneric(bs, null, reg, leftData, rightData);
- }
-
- /** {@inheritDoc} */
- @Override public IgniteBiTuple<RegionProjection, RegionProjection> performSplitGeneric(
- SparseBitSet bs, double[] values, RegionProjection<CategoricalRegionInfo> reg, RegionInfo leftData,
- RegionInfo rightData) {
- int depth = reg.depth();
-
- int lSize = bs.cardinality();
- int rSize = reg.sampleIndexes().length - lSize;
- IgniteBiTuple<Integer[], Integer[]> lrSamples = splitByBitSet(lSize, rSize, reg.sampleIndexes(), bs);
- BitSet leftCats = calculateCats(lrSamples.get1(), values);
- CategoricalRegionInfo lInfo = new CategoricalRegionInfo(leftData.impurity(), leftCats);
-
- // TODO: IGNITE-5892 Check how it will work with sparse data.
- BitSet rightCats = calculateCats(lrSamples.get2(), values);
- CategoricalRegionInfo rInfo = new CategoricalRegionInfo(rightData.impurity(), rightCats);
-
- RegionProjection<CategoricalRegionInfo> rPrj = new RegionProjection<>(lrSamples.get2(), rInfo, depth + 1);
- RegionProjection<CategoricalRegionInfo> lPrj = new RegionProjection<>(lrSamples.get1(), lInfo, depth + 1);
- return new IgniteBiTuple<>(lPrj, rPrj);
- }
-
- /**
- * Powerset iterator. Iterates not over the whole powerset, but on half of it.
- */
- private static class PSI implements Iterator<BitSet> {
-
- /** Current subset number. */
- private int i = 1; // We are not interested in {emptyset, set} split and therefore start from 1.
-
- /** Size of set, subsets of which we iterate over. */
- final int size;
-
- /**
- * @param bitCnt Size of set, subsets of which we iterate over.
- */
- PSI(int bitCnt) {
- this.size = 1 << (bitCnt - 1);
- }
-
- /** {@inheritDoc} */
- @Override public boolean hasNext() {
- return i < size;
- }
-
- /** {@inheritDoc} */
- @Override public BitSet next() {
- BitSet res = BitSet.valueOf(new long[] {i});
- i++;
- return res;
- }
- }
-
- /** */
- private Map<Integer, Integer> mapping(BitSet bs) {
- int bn = 0;
- Map<Integer, Integer> res = new HashMap<>();
-
- int i = 0;
- while ((bn = bs.nextSetBit(bn)) != -1) {
- res.put(bn, i);
- i++;
- bn++;
- }
-
- return res;
- }
-
- /** Get set of categories of given samples */
- private BitSet calculateCats(Integer[] sampleIndexes, double[] values) {
- BitSet res = new BitSet();
-
- for (int smpl : sampleIndexes)
- res.set((int)values[smpl]);
-
- return res;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousFeatureProcessor.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousFeatureProcessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousFeatureProcessor.java
deleted file mode 100644
index 4117993..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousFeatureProcessor.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import java.util.Arrays;
-import java.util.Comparator;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.RegionProjection;
-
-import static org.apache.ignite.ml.trees.trainers.columnbased.vectors.FeatureVectorProcessorUtils.splitByBitSet;
-
-/**
- * Container of projection of samples on continuous feature.
- *
- * @param <D> Information about regions. Designed to contain information which will make computations of impurity
- * optimal.
- */
-public class ContinuousFeatureProcessor<D extends ContinuousRegionInfo> implements
- FeatureProcessor<D, ContinuousSplitInfo<D>> {
- /** ContinuousSplitCalculator used for calculating of best split of each region. */
- private final ContinuousSplitCalculator<D> calc;
-
- /**
- * @param splitCalc Calculator used for calculating splits.
- */
- public ContinuousFeatureProcessor(ContinuousSplitCalculator<D> splitCalc) {
- this.calc = splitCalc;
- }
-
- /** {@inheritDoc} */
- @Override public SplitInfo<D> findBestSplit(RegionProjection<D> ri, double[] values, double[] labels, int regIdx) {
- SplitInfo<D> res = calc.splitRegion(ri.sampleIndexes(), values, labels, regIdx, ri.data());
-
- if (res == null)
- return null;
-
- double lWeight = (double)res.leftData.getSize() / ri.sampleIndexes().length;
- double rWeight = (double)res.rightData.getSize() / ri.sampleIndexes().length;
-
- double infoGain = ri.data().impurity() - lWeight * res.leftData().impurity() - rWeight * res.rightData().impurity();
- res.setInfoGain(infoGain);
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public RegionProjection<D> createInitialRegion(Integer[] samples, double[] values, double[] labels) {
- Arrays.sort(samples, Comparator.comparingDouble(s -> values[s]));
- return new RegionProjection<>(samples, calc.calculateRegionInfo(Arrays.stream(labels), samples.length), 0);
- }
-
- /** {@inheritDoc} */
- @Override public SparseBitSet calculateOwnershipBitSet(RegionProjection<D> reg, double[] values,
- ContinuousSplitInfo<D> s) {
- SparseBitSet res = new SparseBitSet();
-
- for (int i = 0; i < s.leftData().getSize(); i++)
- res.set(reg.sampleIndexes()[i]);
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public IgniteBiTuple<RegionProjection, RegionProjection> performSplit(SparseBitSet bs,
- RegionProjection<D> reg, D leftData, D rightData) {
- int lSize = leftData.getSize();
- int rSize = rightData.getSize();
- int depth = reg.depth();
-
- IgniteBiTuple<Integer[], Integer[]> lrSamples = splitByBitSet(lSize, rSize, reg.sampleIndexes(), bs);
-
- RegionProjection<D> left = new RegionProjection<>(lrSamples.get1(), leftData, depth + 1);
- RegionProjection<D> right = new RegionProjection<>(lrSamples.get2(), rightData, depth + 1);
-
- return new IgniteBiTuple<>(left, right);
- }
-
- /** {@inheritDoc} */
- @Override public IgniteBiTuple<RegionProjection, RegionProjection> performSplitGeneric(SparseBitSet bs,
- double[] labels, RegionProjection<D> reg, RegionInfo leftData, RegionInfo rightData) {
- int lSize = bs.cardinality();
- int rSize = reg.sampleIndexes().length - lSize;
- int depth = reg.depth();
-
- IgniteBiTuple<Integer[], Integer[]> lrSamples = splitByBitSet(lSize, rSize, reg.sampleIndexes(), bs);
-
- D ld = calc.calculateRegionInfo(Arrays.stream(lrSamples.get1()).mapToDouble(s -> labels[s]), lSize);
- D rd = calc.calculateRegionInfo(Arrays.stream(lrSamples.get2()).mapToDouble(s -> labels[s]), rSize);
-
- return new IgniteBiTuple<>(new RegionProjection<>(lrSamples.get1(), ld, depth + 1), new RegionProjection<>(lrSamples.get2(), rd, depth + 1));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousSplitInfo.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousSplitInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousSplitInfo.java
deleted file mode 100644
index 8b45cb5..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousSplitInfo.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.nodes.ContinuousSplitNode;
-import org.apache.ignite.ml.trees.nodes.SplitNode;
-
-/**
- * Information about split of continuous region.
- *
- * @param <D> Class encapsulating information about the region.
- */
-public class ContinuousSplitInfo<D extends RegionInfo> extends SplitInfo<D> {
- /**
- * Threshold used for split.
- * Samples with values less or equal than this go to left region, others go to the right region.
- */
- private final double threshold;
-
- /**
- * @param regionIdx Index of region being split.
- * @param threshold Threshold used for split. Samples with values less or equal than this go to left region, others
- * go to the right region.
- * @param leftData Information about left subregion.
- * @param rightData Information about right subregion.
- */
- public ContinuousSplitInfo(int regionIdx, double threshold, D leftData, D rightData) {
- super(regionIdx, leftData, rightData);
- this.threshold = threshold;
- }
-
- /** {@inheritDoc} */
- @Override public SplitNode createSplitNode(int featureIdx) {
- return new ContinuousSplitNode(threshold, featureIdx);
- }
-
- /**
- * Threshold used for splits.
- * Samples with values less or equal than this go to left region, others go to the right region.
- */
- public double threshold() {
- return threshold;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "ContinuousSplitInfo [" +
- "threshold=" + threshold +
- ", infoGain=" + infoGain +
- ", regionIdx=" + regionIdx +
- ", leftData=" + leftData +
- ", rightData=" + rightData +
- ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureProcessor.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureProcessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureProcessor.java
deleted file mode 100644
index 56508e5..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureProcessor.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.RegionProjection;
-
-/**
- * Base interface for feature processors used in {@link ColumnDecisionTreeTrainer}
- *
- * @param <D> Class representing data of regions resulted from split.
- * @param <S> Class representing data of split.
- */
-public interface FeatureProcessor<D extends RegionInfo, S extends SplitInfo<D>> {
- /**
- * Finds best split by this feature among all splits of all regions.
- *
- * @return best split by this feature among all splits of all regions.
- */
- SplitInfo findBestSplit(RegionProjection<D> regionPrj, double[] values, double[] labels, int regIdx);
-
- /**
- * Creates initial region from samples.
- *
- * @param samples samples.
- * @return region.
- */
- RegionProjection<D> createInitialRegion(Integer[] samples, double[] values, double[] labels);
-
- /**
- * Calculates the bitset mapping each data point to left (corresponding bit is set) or right subregion.
- *
- * @param s data used for calculating the split.
- * @return Bitset mapping each data point to left (corresponding bit is set) or right subregion.
- */
- SparseBitSet calculateOwnershipBitSet(RegionProjection<D> regionPrj, double[] values, S s);
-
- /**
- * Splits given region using bitset which maps data point to left or right subregion.
- * This method is present for the vectors of the same type to be able to pass between them information about regions
- * and therefore used iff the optimal split is received on feature of the same type.
- *
- * @param bs Bitset which maps data point to left or right subregion.
- * @param leftData Data of the left subregion.
- * @param rightData Data of the right subregion.
- * @return This feature vector.
- */
- IgniteBiTuple<RegionProjection, RegionProjection> performSplit(SparseBitSet bs, RegionProjection<D> reg, D leftData,
- D rightData);
-
- /**
- * Splits given region using bitset which maps data point to left or right subregion. This method is used iff the
- * optimal split is received on feature of different type, therefore information about regions is limited to the
- * {@link RegionInfo} class which is base for all classes used to represent region data.
- *
- * @param bs Bitset which maps data point to left or right subregion.
- * @param leftData Data of the left subregion.
- * @param rightData Data of the right subregion.
- * @return This feature vector.
- */
- IgniteBiTuple<RegionProjection, RegionProjection> performSplitGeneric(SparseBitSet bs, double[] values,
- RegionProjection<D> reg, RegionInfo leftData,
- RegionInfo rightData);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureVectorProcessorUtils.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureVectorProcessorUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureVectorProcessorUtils.java
deleted file mode 100644
index 69ff019..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureVectorProcessorUtils.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import org.apache.ignite.lang.IgniteBiTuple;
-
-/** Utility class for feature vector processors. */
-public class FeatureVectorProcessorUtils {
- /**
- * Split target array into two (left and right) arrays by bitset.
- *
- * @param lSize Left array size;
- * @param rSize Right array size.
- * @param samples Arrays to split size.
- * @param bs Bitset specifying split.
- * @return BiTuple containing result of split.
- */
- public static IgniteBiTuple<Integer[], Integer[]> splitByBitSet(int lSize, int rSize, Integer[] samples,
- SparseBitSet bs) {
- Integer[] lArr = new Integer[lSize];
- Integer[] rArr = new Integer[rSize];
-
- int lc = 0;
- int rc = 0;
-
- for (int i = 0; i < lSize + rSize; i++) {
- int si = samples[i];
-
- if (bs.get(si)) {
- lArr[lc] = si;
- lc++;
- }
- else {
- rArr[rc] = si;
- rc++;
- }
- }
-
- return new IgniteBiTuple<>(lArr, rArr);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SampleInfo.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SampleInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SampleInfo.java
deleted file mode 100644
index 8aa4f79..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SampleInfo.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-
-/**
- * Information about given sample within given fixed feature.
- */
-public class SampleInfo implements Externalizable {
- /** Value of projection of this sample on given fixed feature. */
- private double val;
-
- /** Sample index. */
- private int sampleIdx;
-
- /**
- * @param val Value of projection of this sample on given fixed feature.
- * @param sampleIdx Sample index.
- */
- public SampleInfo(double val, int sampleIdx) {
- this.val = val;
- this.sampleIdx = sampleIdx;
- }
-
- /**
- * No-op constructor used for serialization/deserialization.
- */
- public SampleInfo() {
- // No-op.
- }
-
- /**
- * Get the value of projection of this sample on given fixed feature.
- *
- * @return Value of projection of this sample on given fixed feature.
- */
- public double val() {
- return val;
- }
-
- /**
- * Get the sample index.
- *
- * @return Sample index.
- */
- public int sampleInd() {
- return sampleIdx;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeDouble(val);
- out.writeInt(sampleIdx);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- val = in.readDouble();
- sampleIdx = in.readInt();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SplitInfo.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SplitInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SplitInfo.java
deleted file mode 100644
index 124e82f..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SplitInfo.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.nodes.SplitNode;
-
-/**
- * Class encapsulating information about the split.
- *
- * @param <D> Class representing information of left and right subregions.
- */
-public abstract class SplitInfo<D extends RegionInfo> {
- /** Information gain of this split. */
- protected double infoGain;
-
- /** Index of the region to split. */
- protected final int regionIdx;
-
- /** Data of left subregion. */
- protected final D leftData;
-
- /** Data of right subregion. */
- protected final D rightData;
-
- /**
- * Construct the split info.
- *
- * @param regionIdx Index of the region to split.
- * @param leftData Data of left subregion.
- * @param rightData Data of right subregion.
- */
- public SplitInfo(int regionIdx, D leftData, D rightData) {
- this.regionIdx = regionIdx;
- this.leftData = leftData;
- this.rightData = rightData;
- }
-
- /**
- * Index of region to split.
- *
- * @return Index of region to split.
- */
- public int regionIndex() {
- return regionIdx;
- }
-
- /**
- * Information gain of the split.
- *
- * @return Information gain of the split.
- */
- public double infoGain() {
- return infoGain;
- }
-
- /**
- * Data of right subregion.
- *
- * @return Data of right subregion.
- */
- public D rightData() {
- return rightData;
- }
-
- /**
- * Data of left subregion.
- *
- * @return Data of left subregion.
- */
- public D leftData() {
- return leftData;
- }
-
- /**
- * Create SplitNode from this split info.
- *
- * @param featureIdx Index of feature by which goes split.
- * @return SplitNode from this split info.
- */
- public abstract SplitNode createSplitNode(int featureIdx);
-
- /**
- * Set information gain.
- *
- * @param infoGain Information gain.
- */
- public void setInfoGain(double infoGain) {
- this.infoGain = infoGain;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/package-info.java
deleted file mode 100644
index 0dea204..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains feature containers needed by column based decision tree trainers.
- */
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
index e22a3a5..9900f85 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
@@ -28,7 +28,7 @@ import org.apache.ignite.ml.preprocessing.PreprocessingTestSuite;
import org.apache.ignite.ml.regressions.RegressionsTestSuite;
import org.apache.ignite.ml.svm.SVMTestSuite;
import org.apache.ignite.ml.trainers.group.TrainersGroupTestSuite;
-import org.apache.ignite.ml.trees.DecisionTreesTestSuite;
+import org.apache.ignite.ml.tree.DecisionTreeTestSuite;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@@ -41,7 +41,7 @@ import org.junit.runners.Suite;
RegressionsTestSuite.class,
SVMTestSuite.class,
ClusteringTestSuite.class,
- DecisionTreesTestSuite.class,
+ DecisionTreeTestSuite.class,
KNNTestSuite.class,
LocalModelsTest.class,
MLPTestSuite.class,
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistMLPTestUtil.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistMLPTestUtil.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistMLPTestUtil.java
index e624004..d68b355 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistMLPTestUtil.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistMLPTestUtil.java
@@ -25,11 +25,10 @@ import java.util.Random;
import java.util.stream.Stream;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.trees.performance.ColumnDecisionTreeTrainerBenchmark;
import org.apache.ignite.ml.util.MnistUtils;
/** */
-class MnistMLPTestUtil {
+public class MnistMLPTestUtil {
/** Name of the property specifying path to training set images. */
private static final String PROP_TRAINING_IMAGES = "mnist.training.images";
@@ -62,7 +61,7 @@ class MnistMLPTestUtil {
* @return List of MNIST images.
* @throws IOException In case of exception.
*/
- static List<MnistUtils.MnistLabeledImage> loadTrainingSet(int cnt) throws IOException {
+ public static List<MnistUtils.MnistLabeledImage> loadTrainingSet(int cnt) throws IOException {
Properties props = loadMNISTProperties();
return MnistUtils.mnistAsList(props.getProperty(PROP_TRAINING_IMAGES), props.getProperty(PROP_TRAINING_LABELS), new Random(123L), cnt);
}
@@ -74,7 +73,7 @@ class MnistMLPTestUtil {
* @return List of MNIST images.
* @throws IOException In case of exception.
*/
- static List<MnistUtils.MnistLabeledImage> loadTestSet(int cnt) throws IOException {
+ public static List<MnistUtils.MnistLabeledImage> loadTestSet(int cnt) throws IOException {
Properties props = loadMNISTProperties();
return MnistUtils.mnistAsList(props.getProperty(PROP_TEST_IMAGES), props.getProperty(PROP_TEST_LABELS), new Random(123L), cnt);
}
@@ -83,7 +82,7 @@ class MnistMLPTestUtil {
private static Properties loadMNISTProperties() throws IOException {
Properties res = new Properties();
- InputStream is = ColumnDecisionTreeTrainerBenchmark.class.getClassLoader().getResourceAsStream("manualrun/trees/columntrees.manualrun.properties");
+ InputStream is = MnistMLPTestUtil.class.getClassLoader().getResourceAsStream("manualrun/trees/columntrees.manualrun.properties");
res.load(is);
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
new file mode 100644
index 0000000..94bca3f
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.Arrays;
+import java.util.Random;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests for {@link DecisionTreeClassificationTrainer} that require to start the whole Ignite infrastructure.
+ */
+public class DecisionTreeClassificationTrainerIntegrationTest extends GridCommonAbstractTest {
+ /** Number of nodes in grid */
+ private static final int NODE_COUNT = 3;
+
+ /** Ignite instance. */
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ for (int i = 1; i <= NODE_COUNT; i++)
+ startGrid(i);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() {
+ stopAllGrids();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void beforeTest() throws Exception {
+ /* Grid instance. */
+ ignite = grid(NODE_COUNT);
+ ignite.configuration().setPeerClassLoadingEnabled(true);
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+ }
+
+ /** */
+ public void testFit() {
+ int size = 100;
+
+ CacheConfiguration<Integer, double[]> trainingSetCacheCfg = new CacheConfiguration<>();
+ trainingSetCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+ trainingSetCacheCfg.setName("TRAINING_SET");
+
+ IgniteCache<Integer, double[]> data = ignite.createCache(trainingSetCacheCfg);
+
+ Random rnd = new Random(0);
+ for (int i = 0; i < size; i++) {
+ double x = rnd.nextDouble() - 0.5;
+ data.put(i, new double[]{x, x > 0 ? 1 : 0});
+ }
+
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(1, 0);
+
+ DecisionTreeNode tree = trainer.fit(
+ new CacheBasedDatasetBuilder<>(ignite, data),
+ (k, v) -> Arrays.copyOf(v, v.length - 1),
+ (k, v) -> v[v.length - 1]
+ );
+
+ assertTrue(tree instanceof DecisionTreeConditionalNode);
+
+ DecisionTreeConditionalNode node = (DecisionTreeConditionalNode) tree;
+
+ assertEquals(0, node.getThreshold(), 1e-3);
+
+ assertTrue(node.getThenNode() instanceof DecisionTreeLeafNode);
+ assertTrue(node.getElseNode() instanceof DecisionTreeLeafNode);
+
+ DecisionTreeLeafNode thenNode = (DecisionTreeLeafNode) node.getThenNode();
+ DecisionTreeLeafNode elseNode = (DecisionTreeLeafNode) node.getElseNode();
+
+ assertEquals(1, thenNode.getVal(), 1e-10);
+ assertEquals(0, elseNode.getVal(), 1e-10);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
new file mode 100644
index 0000000..2599bfe
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+
+/**
+ * Tests for {@link DecisionTreeClassificationTrainer}.
+ */
+@RunWith(Parameterized.class)
+public class DecisionTreeClassificationTrainerTest {
+ /** Number of parts to be tested. */
+ private static final int[] partsToBeTested = new int[] {1, 2, 3, 4, 5, 7};
+
+ /** Number of partitions. */
+ @Parameterized.Parameter
+ public int parts;
+
+ @Parameterized.Parameters(name = "Data divided on {0} partitions")
+ public static Iterable<Integer[]> data() {
+ List<Integer[]> res = new ArrayList<>();
+ for (int part : partsToBeTested)
+ res.add(new Integer[] {part});
+
+ return res;
+ }
+
+ /** */
+ @Test
+ public void testFit() {
+ int size = 100;
+
+ Map<Integer, double[]> data = new HashMap<>();
+
+ Random rnd = new Random(0);
+ for (int i = 0; i < size; i++) {
+ double x = rnd.nextDouble() - 0.5;
+ data.put(i, new double[]{x, x > 0 ? 1 : 0});
+ }
+
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(1, 0);
+
+ DecisionTreeNode tree = trainer.fit(
+ new LocalDatasetBuilder<>(data, parts),
+ (k, v) -> Arrays.copyOf(v, v.length - 1),
+ (k, v) -> v[v.length - 1]
+ );
+
+ assertTrue(tree instanceof DecisionTreeConditionalNode);
+
+ DecisionTreeConditionalNode node = (DecisionTreeConditionalNode) tree;
+
+ assertEquals(0, node.getThreshold(), 1e-3);
+
+ assertTrue(node.getThenNode() instanceof DecisionTreeLeafNode);
+ assertTrue(node.getElseNode() instanceof DecisionTreeLeafNode);
+
+ DecisionTreeLeafNode thenNode = (DecisionTreeLeafNode) node.getThenNode();
+ DecisionTreeLeafNode elseNode = (DecisionTreeLeafNode) node.getElseNode();
+
+ assertEquals(1, thenNode.getVal(), 1e-10);
+ assertEquals(0, elseNode.getVal(), 1e-10);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
new file mode 100644
index 0000000..754ff20
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.Arrays;
+import java.util.Random;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests for {@link DecisionTreeRegressionTrainer} that require to start the whole Ignite infrastructure.
+ */
+public class DecisionTreeRegressionTrainerIntegrationTest extends GridCommonAbstractTest {
+ /** Number of nodes in grid */
+ private static final int NODE_COUNT = 3;
+
+ /** Ignite instance. */
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ for (int i = 1; i <= NODE_COUNT; i++)
+ startGrid(i);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() {
+ stopAllGrids();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void beforeTest() throws Exception {
+ /* Grid instance. */
+ ignite = grid(NODE_COUNT);
+ ignite.configuration().setPeerClassLoadingEnabled(true);
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+ }
+
+ /** */
+ public void testFit() {
+ int size = 100;
+
+ CacheConfiguration<Integer, double[]> trainingSetCacheCfg = new CacheConfiguration<>();
+ trainingSetCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+ trainingSetCacheCfg.setName("TRAINING_SET");
+
+ IgniteCache<Integer, double[]> data = ignite.createCache(trainingSetCacheCfg);
+
+ Random rnd = new Random(0);
+ for (int i = 0; i < size; i++) {
+ double x = rnd.nextDouble() - 0.5;
+ data.put(i, new double[]{x, x > 0 ? 1 : 0});
+ }
+
+ DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(1, 0);
+
+ DecisionTreeNode tree = trainer.fit(
+ new CacheBasedDatasetBuilder<>(ignite, data),
+ (k, v) -> Arrays.copyOf(v, v.length - 1),
+ (k, v) -> v[v.length - 1]
+ );
+
+ assertTrue(tree instanceof DecisionTreeConditionalNode);
+
+ DecisionTreeConditionalNode node = (DecisionTreeConditionalNode) tree;
+
+ assertEquals(0, node.getThreshold(), 1e-3);
+
+ assertTrue(node.getThenNode() instanceof DecisionTreeLeafNode);
+ assertTrue(node.getElseNode() instanceof DecisionTreeLeafNode);
+
+ DecisionTreeLeafNode thenNode = (DecisionTreeLeafNode) node.getThenNode();
+ DecisionTreeLeafNode elseNode = (DecisionTreeLeafNode) node.getElseNode();
+
+ assertEquals(1, thenNode.getVal(), 1e-10);
+ assertEquals(0, elseNode.getVal(), 1e-10);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
new file mode 100644
index 0000000..3bdbf60
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+
+/**
+ * Tests for {@link DecisionTreeRegressionTrainer}.
+ */
+@RunWith(Parameterized.class)
+public class DecisionTreeRegressionTrainerTest {
+ /** Number of parts to be tested. */
+ private static final int[] partsToBeTested = new int[] {1, 2, 3, 4, 5, 7};
+
+ /** Number of partitions. */
+ @Parameterized.Parameter
+ public int parts;
+
+ @Parameterized.Parameters(name = "Data divided on {0} partitions")
+ public static Iterable<Integer[]> data() {
+ List<Integer[]> res = new ArrayList<>();
+ for (int part : partsToBeTested)
+ res.add(new Integer[] {part});
+
+ return res;
+ }
+
+ /** */
+ @Test
+ public void testFit() {
+ int size = 100;
+
+ Map<Integer, double[]> data = new HashMap<>();
+
+ Random rnd = new Random(0);
+ for (int i = 0; i < size; i++) {
+ double x = rnd.nextDouble() - 0.5;
+ data.put(i, new double[]{x, x > 0 ? 1 : 0});
+ }
+
+ DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(1, 0);
+
+ DecisionTreeNode tree = trainer.fit(
+ new LocalDatasetBuilder<>(data, parts),
+ (k, v) -> Arrays.copyOf(v, v.length - 1),
+ (k, v) -> v[v.length - 1]
+ );
+
+ assertTrue(tree instanceof DecisionTreeConditionalNode);
+
+ DecisionTreeConditionalNode node = (DecisionTreeConditionalNode) tree;
+
+ assertEquals(0, node.getThreshold(), 1e-3);
+
+ assertTrue(node.getThenNode() instanceof DecisionTreeLeafNode);
+ assertTrue(node.getElseNode() instanceof DecisionTreeLeafNode);
+
+ DecisionTreeLeafNode thenNode = (DecisionTreeLeafNode) node.getThenNode();
+ DecisionTreeLeafNode elseNode = (DecisionTreeLeafNode) node.getElseNode();
+
+ assertEquals(1, thenNode.getVal(), 1e-10);
+ assertEquals(0, elseNode.getVal(), 1e-10);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeTestSuite.java
new file mode 100644
index 0000000..2cbb486
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeTestSuite.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import org.apache.ignite.ml.tree.data.DecisionTreeDataTest;
+import org.apache.ignite.ml.tree.impurity.gini.GiniImpurityMeasureCalculatorTest;
+import org.apache.ignite.ml.tree.impurity.gini.GiniImpurityMeasureTest;
+import org.apache.ignite.ml.tree.impurity.mse.MSEImpurityMeasureCalculatorTest;
+import org.apache.ignite.ml.tree.impurity.mse.MSEImpurityMeasureTest;
+import org.apache.ignite.ml.tree.impurity.util.SimpleStepFunctionCompressorTest;
+import org.apache.ignite.ml.tree.impurity.util.StepFunctionTest;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/**
+ * Test suite for all tests located in {@link org.apache.ignite.ml.tree} package.
+ */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ DecisionTreeClassificationTrainerTest.class,
+ DecisionTreeRegressionTrainerTest.class,
+ DecisionTreeClassificationTrainerIntegrationTest.class,
+ DecisionTreeRegressionTrainerIntegrationTest.class,
+ DecisionTreeDataTest.class,
+ GiniImpurityMeasureCalculatorTest.class,
+ GiniImpurityMeasureTest.class,
+ MSEImpurityMeasureCalculatorTest.class,
+ MSEImpurityMeasureTest.class,
+ StepFunctionTest.class,
+ SimpleStepFunctionCompressorTest.class
+})
+public class DecisionTreeTestSuite {
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/data/DecisionTreeDataTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/data/DecisionTreeDataTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/data/DecisionTreeDataTest.java
new file mode 100644
index 0000000..0c89d4e
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/data/DecisionTreeDataTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.data;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link DecisionTreeData}.
+ */
+public class DecisionTreeDataTest {
+ /** */
+ @Test
+ public void testFilter() {
+ double[][] features = new double[][]{{0}, {1}, {2}, {3}, {4}, {5}};
+ double[] labels = new double[]{0, 1, 2, 3, 4, 5};
+
+ DecisionTreeData data = new DecisionTreeData(features, labels);
+ DecisionTreeData filteredData = data.filter(obj -> obj[0] > 2);
+
+ assertArrayEquals(new double[][]{{3}, {4}, {5}}, filteredData.getFeatures());
+ assertArrayEquals(new double[]{3, 4, 5}, filteredData.getLabels(), 1e-10);
+ }
+
+ /** */
+ @Test
+ public void testSort() {
+ double[][] features = new double[][]{{4, 1}, {3, 3}, {2, 0}, {1, 4}, {0, 2}};
+ double[] labels = new double[]{0, 1, 2, 3, 4};
+
+ DecisionTreeData data = new DecisionTreeData(features, labels);
+
+ data.sort(0);
+
+ assertArrayEquals(new double[][]{{0, 2}, {1, 4}, {2, 0}, {3, 3}, {4, 1}}, features);
+ assertArrayEquals(new double[]{4, 3, 2, 1, 0}, labels, 1e-10);
+
+ data.sort(1);
+
+ assertArrayEquals(new double[][]{{2, 0}, {4, 1}, {0, 2}, {3, 3}, {1, 4}}, features);
+ assertArrayEquals(new double[]{2, 0, 4, 1, 3}, labels, 1e-10);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculatorTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculatorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculatorTest.java
new file mode 100644
index 0000000..afd81e8
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculatorTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.gini;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link GiniImpurityMeasureCalculator}.
+ */
+public class GiniImpurityMeasureCalculatorTest {
+ /** */
+ @Test
+ public void testCalculate() {
+ double[][] data = new double[][]{{0, 1}, {1, 0}, {2, 2}, {3, 3}};
+ double[] labels = new double[]{0, 1, 1, 1};
+
+ Map<Double, Integer> encoder = new HashMap<>();
+ encoder.put(0.0, 0);
+ encoder.put(1.0, 1);
+ GiniImpurityMeasureCalculator calculator = new GiniImpurityMeasureCalculator(encoder);
+
+ StepFunction<GiniImpurityMeasure>[] impurity = calculator.calculate(new DecisionTreeData(data, labels));
+
+ assertEquals(2, impurity.length);
+
+ // Check Gini calculated for the first column.
+ assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[0].getX(), 1e-10);
+ assertEquals(-2.500, impurity[0].getY()[0].impurity(), 1e-3);
+ assertEquals(-4.000, impurity[0].getY()[1].impurity(),1e-3);
+ assertEquals(-3.000, impurity[0].getY()[2].impurity(),1e-3);
+ assertEquals(-2.666, impurity[0].getY()[3].impurity(),1e-3);
+ assertEquals(-2.500, impurity[0].getY()[4].impurity(),1e-3);
+
+ // Check Gini calculated for the second column.
+ assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[1].getX(), 1e-10);
+ assertEquals(-2.500, impurity[1].getY()[0].impurity(),1e-3);
+ assertEquals(-2.666, impurity[1].getY()[1].impurity(),1e-3);
+ assertEquals(-3.000, impurity[1].getY()[2].impurity(),1e-3);
+ assertEquals(-2.666, impurity[1].getY()[3].impurity(),1e-3);
+ assertEquals(-2.500, impurity[1].getY()[4].impurity(),1e-3);
+ }
+
+ /** */
+ @Test
+ public void testCalculateWithRepeatedData() {
+ double[][] data = new double[][]{{0}, {1}, {2}, {2}, {3}};
+ double[] labels = new double[]{0, 1, 1, 1, 1};
+
+ Map<Double, Integer> encoder = new HashMap<>();
+ encoder.put(0.0, 0);
+ encoder.put(1.0, 1);
+ GiniImpurityMeasureCalculator calculator = new GiniImpurityMeasureCalculator(encoder);
+
+ StepFunction<GiniImpurityMeasure>[] impurity = calculator.calculate(new DecisionTreeData(data, labels));
+
+ assertEquals(1, impurity.length);
+
+ // Check Gini calculated for the first column.
+ assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[0].getX(), 1e-10);
+ assertEquals(-3.400, impurity[0].getY()[0].impurity(), 1e-3);
+ assertEquals(-5.000, impurity[0].getY()[1].impurity(),1e-3);
+ assertEquals(-4.000, impurity[0].getY()[2].impurity(),1e-3);
+ assertEquals(-3.500, impurity[0].getY()[3].impurity(),1e-3);
+ assertEquals(-3.400, impurity[0].getY()[4].impurity(),1e-3);
+ }
+
+ /** */
+ @Test
+ public void testGetLabelCode() {
+ Map<Double, Integer> encoder = new HashMap<>();
+ encoder.put(0.0, 0);
+ encoder.put(1.0, 1);
+ encoder.put(2.0, 2);
+
+ GiniImpurityMeasureCalculator calculator = new GiniImpurityMeasureCalculator(encoder);
+
+ assertEquals(0, calculator.getLabelCode(0.0));
+ assertEquals(1, calculator.getLabelCode(1.0));
+ assertEquals(2, calculator.getLabelCode(2.0));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureTest.java
new file mode 100644
index 0000000..35c456a
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.gini;
+
+import java.util.Random;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+
+/**
+ * Tests for {@link GiniImpurityMeasure}.
+ */
+public class GiniImpurityMeasureTest {
+ /** */
+ @Test
+ public void testImpurityOnEmptyData() {
+ long[] left = new long[]{0, 0, 0};
+ long[] right = new long[]{0, 0, 0};
+
+ GiniImpurityMeasure impurity = new GiniImpurityMeasure(left, right);
+
+ assertEquals(0.0, impurity.impurity(), 1e-10);
+ }
+
+ /** */
+ @Test
+ public void testImpurityLeftPart() {
+ long[] left = new long[]{3, 0, 0};
+ long[] right = new long[]{0, 0, 0};
+
+ GiniImpurityMeasure impurity = new GiniImpurityMeasure(left, right);
+
+ assertEquals(-3, impurity.impurity(), 1e-10);
+ }
+
+ /** */
+ @Test
+ public void testImpurityRightPart() {
+ long[] left = new long[]{0, 0, 0};
+ long[] right = new long[]{3, 0, 0};
+
+ GiniImpurityMeasure impurity = new GiniImpurityMeasure(left, right);
+
+ assertEquals(-3, impurity.impurity(), 1e-10);
+ }
+
+ /** */
+ @Test
+ public void testImpurityLeftAndRightPart() {
+ long[] left = new long[]{3, 0, 0};
+ long[] right = new long[]{0, 3, 0};
+
+ GiniImpurityMeasure impurity = new GiniImpurityMeasure(left, right);
+
+ assertEquals(-6, impurity.impurity(), 1e-10);
+ }
+
+ /** */
+ @Test
+ public void testAdd() {
+ Random rnd = new Random(0);
+
+ GiniImpurityMeasure a = new GiniImpurityMeasure(
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)},
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)}
+ );
+
+
+ GiniImpurityMeasure b = new GiniImpurityMeasure(
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)},
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)}
+ );
+
+ GiniImpurityMeasure c = a.add(b);
+
+ assertEquals(a.getLeft()[0] + b.getLeft()[0], c.getLeft()[0]);
+ assertEquals(a.getLeft()[1] + b.getLeft()[1], c.getLeft()[1]);
+ assertEquals(a.getLeft()[2] + b.getLeft()[2], c.getLeft()[2]);
+
+ assertEquals(a.getRight()[0] + b.getRight()[0], c.getRight()[0]);
+ assertEquals(a.getRight()[1] + b.getRight()[1], c.getRight()[1]);
+ assertEquals(a.getRight()[2] + b.getRight()[2], c.getRight()[2]);
+ }
+
+ /** */
+ @Test
+ public void testSubtract() {
+ Random rnd = new Random(0);
+
+ GiniImpurityMeasure a = new GiniImpurityMeasure(
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)},
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)}
+ );
+
+
+ GiniImpurityMeasure b = new GiniImpurityMeasure(
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)},
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)}
+ );
+
+ GiniImpurityMeasure c = a.subtract(b);
+
+ assertEquals(a.getLeft()[0] - b.getLeft()[0], c.getLeft()[0]);
+ assertEquals(a.getLeft()[1] - b.getLeft()[1], c.getLeft()[1]);
+ assertEquals(a.getLeft()[2] - b.getLeft()[2], c.getLeft()[2]);
+
+ assertEquals(a.getRight()[0] - b.getRight()[0], c.getRight()[0]);
+ assertEquals(a.getRight()[1] - b.getRight()[1], c.getRight()[1]);
+ assertEquals(a.getRight()[2] - b.getRight()[2], c.getRight()[2]);
+ }
+
+ /** Generates random count. */
+ private long randCnt(Random rnd) {
+ return Math.abs(rnd.nextInt());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculatorTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculatorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculatorTest.java
new file mode 100644
index 0000000..510c18f
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculatorTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.mse;
+
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link MSEImpurityMeasureCalculator}.
+ */
+public class MSEImpurityMeasureCalculatorTest {
+ /** */
+ @Test
+ public void testCalculate() {
+ double[][] data = new double[][]{{0, 2}, {1, 1}, {2, 0}, {3, 3}};
+ double[] labels = new double[]{1, 2, 2, 1};
+
+ MSEImpurityMeasureCalculator calculator = new MSEImpurityMeasureCalculator();
+
+ StepFunction<MSEImpurityMeasure>[] impurity = calculator.calculate(new DecisionTreeData(data, labels));
+
+ assertEquals(2, impurity.length);
+
+ // Test MSE calculated for the first column.
+ assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[0].getX(), 1e-10);
+ assertEquals(1.000, impurity[0].getY()[0].impurity(), 1e-3);
+ assertEquals(0.666, impurity[0].getY()[1].impurity(),1e-3);
+ assertEquals(1.000, impurity[0].getY()[2].impurity(),1e-3);
+ assertEquals(0.666, impurity[0].getY()[3].impurity(),1e-3);
+ assertEquals(1.000, impurity[0].getY()[4].impurity(),1e-3);
+
+ // Test MSE calculated for the second column.
+ assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[1].getX(), 1e-10);
+ assertEquals(1.000, impurity[1].getY()[0].impurity(),1e-3);
+ assertEquals(0.666, impurity[1].getY()[1].impurity(),1e-3);
+ assertEquals(0.000, impurity[1].getY()[2].impurity(),1e-3);
+ assertEquals(0.666, impurity[1].getY()[3].impurity(),1e-3);
+ assertEquals(1.000, impurity[1].getY()[4].impurity(),1e-3);
+ }
+}
[09/14] ignite git commit: ignite-7772 System workers critical
failures handling
Posted by ak...@apache.org.
ignite-7772 System workers critical failures handling
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c807ae95
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c807ae95
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c807ae95
Branch: refs/heads/ignite-8201
Commit: c807ae952c233cf1a8c0a63d543fafe19c40c6aa
Parents: 05d7092
Author: Andrey Kuznetsov <st...@gmail.com>
Authored: Tue Apr 10 17:30:12 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Apr 10 17:30:12 2018 +0300
----------------------------------------------------------------------
.../org/apache/ignite/internal/IgnitionEx.java | 15 +-
.../GridClientConnectionManagerAdapter.java | 6 +
.../impl/GridTcpRouterNioListenerAdapter.java | 6 +
.../discovery/GridDiscoveryManager.java | 16 +-
.../GridCachePartitionExchangeManager.java | 12 +-
.../cache/GridCacheSharedTtlCleanupManager.java | 41 +++--
.../GridCacheDatabaseSharedManager.java | 60 +++++--
.../wal/FileWriteAheadLogManager.java | 157 ++++++++++++-------
.../wal/FsyncModeFileWriteAheadLogManager.java | 34 +++-
.../timeout/GridTimeoutProcessor.java | 102 +++++++-----
.../ignite/internal/util/StripedExecutor.java | 69 +++++---
.../ignite/internal/util/nio/GridNioServer.java | 43 ++++-
.../util/nio/GridNioServerListener.java | 6 +
.../util/nio/GridNioServerListenerAdapter.java | 6 +
.../communication/tcp/TcpCommunicationSpi.java | 41 ++++-
.../ignite/spi/discovery/tcp/ServerImpl.java | 51 +++++-
.../internal/util/StripedExecutorTest.java | 2 +-
17 files changed, 501 insertions(+), 166 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index 417ba1e..10a0752 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -137,6 +137,7 @@ import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
import static org.apache.ignite.configuration.IgniteConfiguration.DFLT_THREAD_KEEP_ALIVE_TIME;
import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEMORY_POLICY_MAX_SIZE;
import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEM_PLC_DEFAULT_NAME;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.IgniteComponentType.SPRING;
import static org.apache.ignite.plugin.segmentation.SegmentationPolicy.RESTART_JVM;
@@ -1806,7 +1807,13 @@ public class IgnitionEx {
cfg.getStripedPoolSize(),
cfg.getIgniteInstanceName(),
"sys",
- log);
+ log,
+ new Thread.UncaughtExceptionHandler() {
+ @Override public void uncaughtException(Thread thread, Throwable t) {
+ if (grid != null)
+ grid.context().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, t));
+ }
+ });
// Note that since we use 'LinkedBlockingQueue', number of
// maximum threads has no effect.
@@ -1846,6 +1853,12 @@ public class IgnitionEx {
cfg.getIgniteInstanceName(),
"data-streamer",
log,
+ new Thread.UncaughtExceptionHandler() {
+ @Override public void uncaughtException(Thread thread, Throwable t) {
+ if (grid != null)
+ grid.context().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, t));
+ }
+ },
true);
// Note that we do not pre-start threads here as igfs pool may not be needed.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
index 829b188..fe0453f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
@@ -38,6 +38,7 @@ import java.util.logging.Logger;
import javax.net.ssl.SSLContext;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.client.GridClientClosedException;
import org.apache.ignite.internal.client.GridClientConfiguration;
import org.apache.ignite.internal.client.GridClientException;
@@ -656,6 +657,11 @@ public abstract class GridClientConnectionManagerAdapter implements GridClientCo
}
}
+ /** {@inheritDoc} */
+ @Override public void onFailure(FailureType failureType, Throwable failure) {
+ // No-op.
+ }
+
/**
* Handles client handshake response.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java
index 22f5152..75aa6f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.client.GridClientException;
import org.apache.ignite.internal.client.GridClientFuture;
import org.apache.ignite.internal.client.GridClientFutureListener;
@@ -191,6 +192,11 @@ public abstract class GridTcpRouterNioListenerAdapter implements GridNioServerLi
}
/** {@inheritDoc} */
+ @Override public void onFailure(FailureType failureType, Throwable failure) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
@Override public void onSessionWriteTimeout(GridNioSession ses) {
U.warn(log, "Closing NIO session because of write timeout.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 4c5690e..b0d3256 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -147,6 +147,8 @@ import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED;
import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DATA_REGIONS_OFFHEAP_SIZE;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DEPLOYMENT_MODE;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT;
@@ -2669,13 +2671,21 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
body0();
}
catch (InterruptedException e) {
+ if (!isCancelled)
+ ctx.failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, e));
+
throw e;
}
catch (Throwable t) {
- U.error(log, "Unexpected exception in discovery worker thread (ignored).", t);
+ U.error(log, "Exception in discovery worker thread.", t);
+
+ if (t instanceof Error) {
+ FailureType type = t instanceof OutOfMemoryError ? CRITICAL_ERROR : SYSTEM_WORKER_TERMINATION;
- if (t instanceof Error)
- throw (Error)t;
+ ctx.failure().process(new FailureContext(type, t));
+
+ throw t;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 77ffce3..e40493f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -123,6 +123,7 @@ import static org.apache.ignite.IgniteSystemProperties.getLong;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.GridTopic.TOPIC_CACHE;
import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
@@ -2274,11 +2275,20 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
try {
body0();
}
+ catch (InterruptedException | IgniteInterruptedCheckedException e) {
+ if (!stop)
+ err = e;
+ }
catch (Throwable e) {
err = e;
}
finally {
- if (!stop)
+ if (err == null && !stop)
+ err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly");
+
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
index 8f3d738..613e93b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
@@ -20,11 +20,15 @@ package org.apache.ignite.internal.processors.cache;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.failure.FailureContext;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.util.worker.GridWorker;
import org.apache.ignite.thread.IgniteThread;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
+
/**
* Periodically removes expired entities from caches with {@link CacheConfiguration#isEagerTtl()} flag set.
*/
@@ -122,19 +126,38 @@ public class GridCacheSharedTtlCleanupManager extends GridCacheSharedManagerAdap
/** {@inheritDoc} */
@Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
- while (!isCancelled()) {
- boolean expiredRemains = false;
+ Throwable err = null;
+
+ try {
+ while (!isCancelled()) {
+ boolean expiredRemains = false;
+
+ for (GridCacheTtlManager mgr : mgrs) {
+ if (mgr.expire(CLEANUP_WORKER_ENTRIES_PROCESS_LIMIT))
+ expiredRemains = true;
- for (GridCacheTtlManager mgr : mgrs) {
- if (mgr.expire(CLEANUP_WORKER_ENTRIES_PROCESS_LIMIT))
- expiredRemains = true;
+ if (isCancelled())
+ return;
+ }
- if (isCancelled())
- return;
+ if (!expiredRemains)
+ U.sleep(CLEANUP_WORKER_SLEEP_INTERVAL);
}
+ }
+ catch (Throwable t) {
+ if (!(t instanceof IgniteInterruptedCheckedException))
+ err = t;
- if (!expiredRemains)
- U.sleep(CLEANUP_WORKER_SLEEP_INTERVAL);
+ throw t;
+ }
+ finally {
+ if (err == null && !isCancelled)
+ err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly");
+
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 70fc688..caf27b7 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -165,6 +165,8 @@ import static java.nio.file.StandardOpenOption.READ;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_ID;
/**
@@ -2787,32 +2789,58 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/** {@inheritDoc} */
- @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
- while (!isCancelled()) {
- waitCheckpointEvent();
+ @Override protected void body() {
+ Throwable err = null;
- GridFutureAdapter<Void> enableChangeApplied = GridCacheDatabaseSharedManager.this.enableChangeApplied;
+ try {
+ while (!isCancelled()) {
+ waitCheckpointEvent();
- if (enableChangeApplied != null) {
- enableChangeApplied.onDone();
+ GridFutureAdapter<Void> enableChangeApplied = GridCacheDatabaseSharedManager.this.enableChangeApplied;
- GridCacheDatabaseSharedManager.this.enableChangeApplied = null;
- }
+ if (enableChangeApplied != null) {
+ enableChangeApplied.onDone();
- if (checkpointsEnabled)
- doCheckpoint();
- else {
- synchronized (this) {
- scheduledCp.nextCpTs = U.currentTimeMillis() + checkpointFreq;
+ GridCacheDatabaseSharedManager.this.enableChangeApplied = null;
+ }
+
+ if (checkpointsEnabled)
+ doCheckpoint();
+ else {
+ synchronized (this) {
+ scheduledCp.nextCpTs = U.currentTimeMillis() + checkpointFreq;
+ }
}
}
}
+ catch (Throwable t) {
+ err = t;
+
+ scheduledCp.cpFinishFut.onDone(t);
+
+ throw t;
+ }
+ finally {
+ if (err == null && !(stopping && isCancelled))
+ err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly");
+
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+ }
// Final run after the cancellation.
- if (checkpointsEnabled && !shutdownNow)
- doCheckpoint();
+ if (checkpointsEnabled && !shutdownNow) {
+ try {
+ doCheckpoint();
- scheduledCp.cpFinishFut.onDone(new NodeStoppingException("Node is stopping."));
+ scheduledCp.cpFinishFut.onDone(new NodeStoppingException("Node is stopping."));
+ }
+ catch (Throwable e) {
+ scheduledCp.cpFinishFut.onDone(e);
+ }
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 2fff481..a40811b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -117,6 +117,8 @@ import static java.nio.file.StandardOpenOption.WRITE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.SWITCH_SEGMENT_RECORD;
import static org.apache.ignite.internal.processors.cache.persistence.wal.SegmentedRingByteBuffer.BufferMode.DIRECT;
import static org.apache.ignite.internal.util.IgniteUtils.findField;
@@ -682,7 +684,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (IgniteCheckedException e) {
U.error(log, "Unable to perform segment rollover: " + e.getMessage(), e);
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
}
}
@@ -1234,7 +1236,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (IOException e) {
StorageException se = new StorageException("Unable to initialize WAL segment", e);
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, se));
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
throw se;
}
@@ -1499,6 +1501,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
}
}
+ Throwable err = null;
+
try {
synchronized (this) {
while (curAbsWalIdx == -1 && !stopped)
@@ -1560,6 +1564,18 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
}
+ catch (Throwable t) {
+ err = t;
+ }
+ finally {
+ if (err == null && !stopped)
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
+
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+ }
}
/**
@@ -1884,8 +1900,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
}
catch (IgniteCheckedException | IOException e) {
U.error(log, "Unexpected error during WAL compression", e);
-
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
}
catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
@@ -2005,6 +2019,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/** {@inheritDoc} */
@Override public void run() {
+ Throwable err = null;
+
while (!Thread.currentThread().isInterrupted() && !stopped) {
try {
long segmentToDecompress = segmentsQueue.take();
@@ -2034,10 +2050,17 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
}
- catch (IOException e) {
- U.error(log, "Unexpected error during WAL decompression", e);
+ catch (Throwable t) {
+ err = t;
+ }
+ finally {
+ if (err == null && !stopped)
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
}
}
}
@@ -3146,78 +3169,94 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/** {@inheritDoc} */
@Override public void run() {
- while (!shutdown && !Thread.currentThread().isInterrupted()) {
- while (waiters.isEmpty()) {
- if (!shutdown)
- LockSupport.park();
- else {
- unparkWaiters(Long.MAX_VALUE);
-
- return;
- }
- }
+ Throwable err = null;
- Long pos = null;
+ try {
+ while (!shutdown && !Thread.currentThread().isInterrupted()) {
+ while (waiters.isEmpty()) {
+ if (!shutdown)
+ LockSupport.park();
+ else {
+ unparkWaiters(Long.MAX_VALUE);
- for (Long val : waiters.values()) {
- if (val > Long.MIN_VALUE)
- pos = val;
- }
+ return;
+ }
+ }
- if (pos == null)
- continue;
- else if (pos < UNCONDITIONAL_FLUSH) {
- try {
- assert pos == FILE_CLOSE || pos == FILE_FORCE : pos;
+ Long pos = null;
- if (pos == FILE_CLOSE)
- currHnd.fileIO.close();
- else if (pos == FILE_FORCE)
- currHnd.fileIO.force();
+ for (Long val : waiters.values()) {
+ if (val > Long.MIN_VALUE)
+ pos = val;
}
- catch (IOException e) {
- log.error("Exception in WAL writer thread: ", e);
- err = e;
+ if (pos == null)
+ continue;
+ else if (pos < UNCONDITIONAL_FLUSH) {
+ try {
+ assert pos == FILE_CLOSE || pos == FILE_FORCE : pos;
- unparkWaiters(Long.MAX_VALUE);
+ if (pos == FILE_CLOSE)
+ currHnd.fileIO.close();
+ else if (pos == FILE_FORCE)
+ currHnd.fileIO.force();
+ }
+ catch (IOException e) {
+ log.error("Exception in WAL writer thread: ", e);
- return;
- }
+ err = e;
- unparkWaiters(pos);
- }
+ unparkWaiters(Long.MAX_VALUE);
- List<SegmentedRingByteBuffer.ReadSegment> segs = currentHandle().buf.poll(pos);
+ return;
+ }
- if (segs == null) {
- unparkWaiters(pos);
+ unparkWaiters(pos);
+ }
- continue;
- }
+ List<SegmentedRingByteBuffer.ReadSegment> segs = currentHandle().buf.poll(pos);
- for (int i = 0; i < segs.size(); i++) {
- SegmentedRingByteBuffer.ReadSegment seg = segs.get(i);
+ if (segs == null) {
+ unparkWaiters(pos);
- try {
- writeBuffer(seg.position(), seg.buffer());
+ continue;
}
- catch (Throwable e) {
- log.error("Exception in WAL writer thread: ", e);
- err = e;
- }
- finally {
- seg.release();
+ for (int i = 0; i < segs.size(); i++) {
+ SegmentedRingByteBuffer.ReadSegment seg = segs.get(i);
+
+ try {
+ writeBuffer(seg.position(), seg.buffer());
+ }
+ catch (Throwable e) {
+ log.error("Exception in WAL writer thread: ", e);
+
+ err = e;
+ }
+ finally {
+ seg.release();
- long p = pos <= UNCONDITIONAL_FLUSH || err != null ? Long.MAX_VALUE : currentHandle().written;
+ long p = pos <= UNCONDITIONAL_FLUSH || err != null ? Long.MAX_VALUE : currentHandle().written;
- unparkWaiters(p);
+ unparkWaiters(p);
+ }
}
}
+
+ unparkWaiters(Long.MAX_VALUE);
}
+ catch (Throwable t) {
+ err = t;
+ }
+ finally {
+ if (err == null && !shutdown)
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
- unparkWaiters(Long.MAX_VALUE);
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+ }
}
/**
@@ -3283,7 +3322,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
Throwable err = walWriter.err;
if (err != null)
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, err));
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
if (expPos == UNCONDITIONAL_FLUSH)
expPos = (currentHandle().buf.tail());
@@ -3372,7 +3411,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (IOException e) {
StorageException se = new StorageException("Unable to write", e);
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, se));
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
throw se;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 59196bb..c7d2c11 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -110,6 +110,7 @@ import static java.nio.file.StandardOpenOption.READ;
import static java.nio.file.StandardOpenOption.WRITE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
/**
* File WAL manager.
@@ -1338,6 +1339,8 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
}
}
+ Throwable err = null;
+
try {
synchronized (this) {
while (curAbsWalIdx == -1 && !stopped)
@@ -1399,6 +1402,18 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
}
+ catch (Throwable t) {
+ err = t;
+ }
+ finally {
+ if (err == null && !stopped)
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
+
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+ }
}
/**
@@ -1721,8 +1736,6 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
}
catch (IgniteCheckedException | IOException e) {
U.error(log, "Unexpected error during WAL compression", e);
-
- cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
@@ -1814,6 +1827,8 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
/** {@inheritDoc} */
@Override public void run() {
+ Throwable err = null;
+
while (!Thread.currentThread().isInterrupted() && !stopped) {
try {
long segmentToDecompress = segmentsQueue.take();
@@ -1840,13 +1855,20 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
decompressionFutures.remove(segmentToDecompress).onDone();
}
}
- catch (InterruptedException e){
+ catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
}
- catch (IOException e) {
- U.error(log, "Unexpected error during WAL decompression", e);
+ catch (Throwable t) {
+ err = t;
+ }
+ finally {
+ if (err == null && !stopped)
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
- cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
index ff6beb4..a09d6fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
@@ -21,6 +21,7 @@ import java.io.Closeable;
import java.util.Comparator;
import java.util.Iterator;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.failure.FailureContext;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.processors.GridProcessorAdapter;
import org.apache.ignite.internal.util.GridConcurrentSkipListSet;
@@ -32,6 +33,9 @@ import org.apache.ignite.internal.util.worker.GridWorker;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.thread.IgniteThread;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
+
/**
* Detects timeout events and processes them.
*/
@@ -146,61 +150,81 @@ public class GridTimeoutProcessor extends GridProcessorAdapter {
/** {@inheritDoc} */
@Override protected void body() throws InterruptedException {
- while (!isCancelled()) {
- long now = U.currentTimeMillis();
+ Throwable err = null;
- for (Iterator<GridTimeoutObject> iter = timeoutObjs.iterator(); iter.hasNext();) {
- GridTimeoutObject timeoutObj = iter.next();
+ try {
+ while (!isCancelled()) {
+ long now = U.currentTimeMillis();
- if (timeoutObj.endTime() <= now) {
- try {
- boolean rmvd = timeoutObjs.remove(timeoutObj);
+ for (Iterator<GridTimeoutObject> iter = timeoutObjs.iterator(); iter.hasNext(); ) {
+ GridTimeoutObject timeoutObj = iter.next();
- if (log.isDebugEnabled())
- log.debug("Timeout has occurred [obj=" + timeoutObj + ", process=" + rmvd + ']');
+ if (timeoutObj.endTime() <= now) {
+ try {
+ boolean rmvd = timeoutObjs.remove(timeoutObj);
- if (rmvd)
- timeoutObj.onTimeout();
- }
- catch (Throwable e) {
- if (isCancelled() && !(e instanceof Error)){
if (log.isDebugEnabled())
- log.debug("Error when executing timeout callback: " + timeoutObj);
+ log.debug("Timeout has occurred [obj=" + timeoutObj + ", process=" + rmvd + ']');
- return;
+ if (rmvd)
+ timeoutObj.onTimeout();
}
+ catch (Throwable e) {
+ if (isCancelled() && !(e instanceof Error)) {
+ if (log.isDebugEnabled())
+ log.debug("Error when executing timeout callback: " + timeoutObj);
- U.error(log, "Error when executing timeout callback: " + timeoutObj, e);
+ return;
+ }
- if (e instanceof Error)
- throw e;
+ U.error(log, "Error when executing timeout callback: " + timeoutObj, e);
+
+ if (e instanceof Error)
+ throw e;
+ }
}
+ else
+ break;
}
- else
- break;
- }
-
- synchronized (mux) {
- while (!isCancelled()) {
- // Access of the first element must be inside of
- // synchronization block, so we don't miss out
- // on thread notification events sent from
- // 'addTimeoutObject(..)' method.
- GridTimeoutObject first = timeoutObjs.firstx();
-
- if (first != null) {
- long waitTime = first.endTime() - U.currentTimeMillis();
- if (waitTime > 0)
- mux.wait(waitTime);
+ synchronized (mux) {
+ while (!isCancelled()) {
+ // Access of the first element must be inside of
+ // synchronization block, so we don't miss out
+ // on thread notification events sent from
+ // 'addTimeoutObject(..)' method.
+ GridTimeoutObject first = timeoutObjs.firstx();
+
+ if (first != null) {
+ long waitTime = first.endTime() - U.currentTimeMillis();
+
+ if (waitTime > 0)
+ mux.wait(waitTime);
+ else
+ break;
+ }
else
- break;
+ mux.wait(5000);
}
- else
- mux.wait(5000);
}
}
}
+ catch (Throwable t) {
+ if (!(t instanceof InterruptedException))
+ err = t;
+
+ throw t;
+ }
+ finally {
+ if (err == null && !isCancelled)
+ err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly.");
+
+ if (err instanceof OutOfMemoryError)
+ ctx.failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ ctx.failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+ }
+
}
}
@@ -284,4 +308,4 @@ public class GridTimeoutProcessor extends GridProcessorAdapter {
return S.toString(CancelableTask.class, this);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
index 630d34c..c6383ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
@@ -64,9 +64,11 @@ public class StripedExecutor implements ExecutorService {
* @param igniteInstanceName Node name.
* @param poolName Pool name.
* @param log Logger.
+ * @param errHnd Exception handler.
*/
- public StripedExecutor(int cnt, String igniteInstanceName, String poolName, final IgniteLogger log) {
- this(cnt, igniteInstanceName, poolName, log, false);
+ public StripedExecutor(int cnt, String igniteInstanceName, String poolName, final IgniteLogger log,
+ Thread.UncaughtExceptionHandler errHnd) {
+ this(cnt, igniteInstanceName, poolName, log, errHnd, false);
}
/**
@@ -74,9 +76,11 @@ public class StripedExecutor implements ExecutorService {
* @param igniteInstanceName Node name.
* @param poolName Pool name.
* @param log Logger.
+ * @param errHnd Exception handler.
* @param stealTasks {@code True} to steal tasks.
*/
- public StripedExecutor(int cnt, String igniteInstanceName, String poolName, final IgniteLogger log, boolean stealTasks) {
+ public StripedExecutor(int cnt, String igniteInstanceName, String poolName, final IgniteLogger log,
+ Thread.UncaughtExceptionHandler errHnd, boolean stealTasks) {
A.ensure(cnt > 0, "cnt > 0");
boolean success = false;
@@ -91,15 +95,9 @@ public class StripedExecutor implements ExecutorService {
try {
for (int i = 0; i < cnt; i++) {
- stripes[i] = stealTasks ? new StripeConcurrentQueue(
- igniteInstanceName,
- poolName,
- i,
- log, stripes) : new StripeConcurrentQueue(
- igniteInstanceName,
- poolName,
- i,
- log);
+ stripes[i] = stealTasks
+ ? new StripeConcurrentQueue(igniteInstanceName, poolName, i, log, stripes, errHnd)
+ : new StripeConcurrentQueue(igniteInstanceName, poolName, i, log, errHnd);
}
for (int i = 0; i < cnt; i++)
@@ -434,22 +432,28 @@ public class StripedExecutor implements ExecutorService {
/** Thread executing the loop. */
protected Thread thread;
+ /** Exception handler. */
+ private Thread.UncaughtExceptionHandler errHnd;
+
/**
* @param igniteInstanceName Ignite instance name.
* @param poolName Pool name.
* @param idx Stripe index.
* @param log Logger.
+ * @param errHnd Exception handler.
*/
public Stripe(
String igniteInstanceName,
String poolName,
int idx,
- IgniteLogger log
+ IgniteLogger log,
+ Thread.UncaughtExceptionHandler errHnd
) {
this.igniteInstanceName = igniteInstanceName;
this.poolName = poolName;
this.idx = idx;
this.log = log;
+ this.errHnd = errHnd;
}
/**
@@ -463,6 +467,8 @@ public class StripedExecutor implements ExecutorService {
idx,
GridIoPolicy.UNDEFINED);
+ thread.setUncaughtExceptionHandler(errHnd);
+
thread.start();
}
@@ -518,9 +524,19 @@ public class StripedExecutor implements ExecutorService {
return;
}
catch (Throwable e) {
+ if (e instanceof OutOfMemoryError) {
+ // Re-throwing to exploit uncaught exception handler.
+ throw e;
+ }
+
U.error(log, "Failed to execute runnable.", e);
}
}
+
+ if (!stopping) {
+ throw new IllegalStateException("Thread " + Thread.currentThread().getName() +
+ " is terminated unexpectedly");
+ }
}
/**
@@ -576,14 +592,16 @@ public class StripedExecutor implements ExecutorService {
* @param poolName Pool name.
* @param idx Stripe index.
* @param log Logger.
+ * @param errHnd Exception handler.
*/
StripeConcurrentQueue(
String igniteInstanceName,
String poolName,
int idx,
- IgniteLogger log
+ IgniteLogger log,
+ Thread.UncaughtExceptionHandler errHnd
) {
- this(igniteInstanceName, poolName, idx, log, null);
+ this(igniteInstanceName, poolName, idx, log, null, errHnd);
}
/**
@@ -591,19 +609,22 @@ public class StripedExecutor implements ExecutorService {
* @param poolName Pool name.
* @param idx Stripe index.
* @param log Logger.
+ * @param errHnd Exception handler.
*/
StripeConcurrentQueue(
String igniteInstanceName,
String poolName,
int idx,
IgniteLogger log,
- Stripe[] others
+ Stripe[] others,
+ Thread.UncaughtExceptionHandler errHnd
) {
super(
igniteInstanceName,
poolName,
idx,
- log);
+ log,
+ errHnd);
this.others = others;
@@ -702,17 +723,20 @@ public class StripedExecutor implements ExecutorService {
* @param poolName Pool name.
* @param idx Stripe index.
* @param log Logger.
+ * @param errHnd Exception handler.
*/
public StripeConcurrentQueueNoPark(
String igniteInstanceName,
String poolName,
int idx,
- IgniteLogger log
+ IgniteLogger log,
+ Thread.UncaughtExceptionHandler errHnd
) {
super(igniteInstanceName,
poolName,
idx,
- log);
+ log,
+ errHnd);
}
/** {@inheritDoc} */
@@ -758,17 +782,20 @@ public class StripedExecutor implements ExecutorService {
* @param poolName Pool name.
* @param idx Stripe index.
* @param log Logger.
+ * @param errHnd Exception handler.
*/
public StripeConcurrentBlockingQueue(
String igniteInstanceName,
String poolName,
int idx,
- IgniteLogger log
+ IgniteLogger log,
+ Thread.UncaughtExceptionHandler errHnd
) {
super(igniteInstanceName,
poolName,
idx,
- log);
+ log,
+ errHnd);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index 0fcde0e..3597a05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -77,6 +77,8 @@ import org.apache.ignite.plugin.extensions.communication.MessageWriter;
import org.apache.ignite.thread.IgniteThread;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.MSG_WRITER;
import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.NIO_OPERATION;
@@ -1749,6 +1751,8 @@ public class GridNioServer<T> {
/** {@inheritDoc} */
@Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
+ Throwable err = null;
+
try {
boolean reset = false;
@@ -1774,9 +1778,24 @@ public class GridNioServer<T> {
catch (Throwable e) {
U.error(log, "Caught unhandled exception in NIO worker thread (restart the node).", e);
+ err = e;
+
if (e instanceof Error)
throw e;
}
+ finally {
+ if (err instanceof OutOfMemoryError)
+ lsnr.onFailure(CRITICAL_ERROR, err);
+ else if (!closed) {
+ if (err == null)
+ lsnr.onFailure(SYSTEM_WORKER_TERMINATION,
+ new IllegalStateException("Thread " + name() + " is terminated unexpectedly"));
+ else if (err instanceof InterruptedException)
+ lsnr.onFailure(SYSTEM_WORKER_TERMINATION, err);
+ }
+ else if (err != null)
+ lsnr.onFailure(SYSTEM_WORKER_TERMINATION, err);
+ }
}
/**
@@ -2790,6 +2809,8 @@ public class GridNioServer<T> {
/** {@inheritDoc} */
@Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
+ Throwable err = null;
+
try {
boolean reset = false;
@@ -2812,8 +2833,28 @@ public class GridNioServer<T> {
}
}
}
+ catch (Throwable t) {
+ if (!(t instanceof IgniteInterruptedCheckedException))
+ err = t;
+
+ throw t;
+ }
finally {
- closeSelector(); // Safety.
+ try {
+ closeSelector(); // Safety.
+ }
+ catch (RuntimeException ignore) {
+ // No-op.
+ }
+
+ if (err == null && !closed)
+ err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly");
+
+ if (err instanceof OutOfMemoryError)
+ lsnr.onFailure(CRITICAL_ERROR, err);
+ else if (err != null)
+ lsnr.onFailure(SYSTEM_WORKER_TERMINATION, err);
+
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListener.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListener.java
index db28792..14c5a74 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListener.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.util.nio;
+import org.apache.ignite.failure.FailureType;
import org.jetbrains.annotations.Nullable;
/**
@@ -69,4 +70,9 @@ public interface GridNioServerListener<T> {
* @param ses Session that is idle.
*/
public void onSessionIdleTimeout(GridNioSession ses);
+
+ /**
+ * Called when critical failure occurs in server implementation.
+ */
+ public void onFailure(FailureType failureType, Throwable failure);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListenerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListenerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListenerAdapter.java
index 5d222c1..b6b20b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListenerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListenerAdapter.java
@@ -17,6 +17,8 @@
package org.apache.ignite.internal.util.nio;
+import org.apache.ignite.failure.FailureType;
+
/**
* Server listener adapter providing empty methods implementation for rarely used methods.
*/
@@ -35,4 +37,8 @@ public abstract class GridNioServerListenerAdapter<T> implements GridNioServerLi
@Override public void onMessageSent(GridNioSession ses, T msg) {
// No-op.
}
+
+ @Override public void onFailure(FailureType failureType, Throwable failure) {
+ // No-op.
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 4a0710e..9e7b592 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -62,7 +62,10 @@ import org.apache.ignite.configuration.AddressResolver;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.Event;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.IgniteKernal;
@@ -151,6 +154,8 @@ import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.SSL_META;
import static org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture.SES_FUT_META;
import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.ALREADY_CONNECTED;
@@ -798,6 +803,11 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
}
}
+ /** {@inheritDoc} */
+ @Override public void onFailure(FailureType failureType, Throwable failure) {
+ ((IgniteEx)ignite).context().failure().process(new FailureContext(failureType, failure));
+ }
+
/**
* @param recovery Recovery descriptor.
* @param ses Session.
@@ -4190,13 +4200,32 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
if (log.isDebugEnabled())
log.debug("Tcp communication worker has been started.");
- while (!isInterrupted()) {
- DisconnectedSessionInfo disconnectData = q.poll(idleConnTimeout, TimeUnit.MILLISECONDS);
+ Throwable err = null;
- if (disconnectData != null)
- processDisconnect(disconnectData);
- else
- processIdle();
+ try {
+ while (!isInterrupted()) {
+ DisconnectedSessionInfo disconnectData = q.poll(idleConnTimeout, TimeUnit.MILLISECONDS);
+
+ if (disconnectData != null)
+ processDisconnect(disconnectData);
+ else
+ processIdle();
+ }
+ }
+ catch (Throwable t) {
+ if (!(t instanceof InterruptedException))
+ err = t;
+
+ throw t;
+ }
+ finally {
+ if (err == null && !stopping)
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly.");
+
+ if (err instanceof OutOfMemoryError)
+ ((IgniteEx)ignite).context().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ ((IgniteEx)ignite).context().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 4aa1316..7bf37e1 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -50,6 +50,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.LinkedBlockingDeque;
@@ -66,6 +67,8 @@ import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cache.CacheMetrics;
import org.apache.ignite.cluster.ClusterMetrics;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.IgniteNodeAttributes;
@@ -73,6 +76,7 @@ import org.apache.ignite.internal.IgnitionEx;
import org.apache.ignite.internal.events.DiscoveryCustomEvent;
import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper;
import org.apache.ignite.internal.managers.discovery.DiscoveryServerOnlyCustomMessage;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
import org.apache.ignite.internal.processors.security.SecurityContext;
import org.apache.ignite.internal.processors.security.SecurityUtils;
import org.apache.ignite.internal.util.GridBoundedLinkedHashSet;
@@ -137,7 +141,6 @@ import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryServerOnlyCustom
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryStatusCheckMessage;
import org.apache.ignite.thread.IgniteThreadPoolExecutor;
import org.jetbrains.annotations.Nullable;
-import java.util.concurrent.ConcurrentHashMap;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCOVERY_CLIENT_RECONNECT_HISTORY_SIZE;
@@ -149,6 +152,8 @@ import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED;
import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER;
@@ -2609,12 +2614,20 @@ class ServerImpl extends TcpDiscoveryImpl {
/** {@inheritDoc} */
@Override protected void body() throws InterruptedException {
+ Throwable err = null;
+
try {
super.body();
}
+ catch (InterruptedException e) {
+ if (!spi.isNodeStopping0())
+ err = e;
+
+ throw e;
+ }
catch (Throwable e) {
if (!spi.isNodeStopping0() && spiStateCopy() != DISCONNECTING) {
- final Ignite ignite = spi.ignite();
+ final Ignite ignite = spi.ignite();
if (ignite != null) {
U.error(log, "TcpDiscoverSpi's message worker thread failed abnormally. " +
@@ -2637,9 +2650,22 @@ class ServerImpl extends TcpDiscoveryImpl {
}
}
+ err = e;
+
// Must be processed by IgniteSpiThread as well.
throw e;
}
+ finally {
+ if (err == null && !spi.isNodeStopping0())
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly.");
+
+ FailureProcessor failure = ((IgniteEx)spi.ignite()).context().failure();
+
+ if (err instanceof OutOfMemoryError)
+ failure.process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ failure.process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+ }
}
/**
@@ -5597,7 +5623,9 @@ class ServerImpl extends TcpDiscoveryImpl {
}
/** {@inheritDoc} */
- @Override protected void body() throws InterruptedException {
+ @Override protected void body() {
+ Throwable err = null;
+
try {
while (!isInterrupted()) {
Socket sock = srvrSock.accept();
@@ -5630,13 +5658,30 @@ class ServerImpl extends TcpDiscoveryImpl {
onException("Failed to accept TCP connection.", e);
if (!isInterrupted()) {
+ err = e;
+
if (U.isMacInvalidArgumentError(e))
U.error(log, "Failed to accept TCP connection\n\t" + U.MAC_INVALID_ARG_MSG, e);
else
U.error(log, "Failed to accept TCP connection.", e);
}
}
+ catch (Throwable t) {
+ err = t;
+
+ throw t;
+ }
finally {
+ if (err == null && !spi.isNodeStopping0())
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly.");
+
+ FailureProcessor failure = ((IgniteEx)spi.ignite()).context().failure();
+
+ if (err instanceof OutOfMemoryError)
+ failure.process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ failure.process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+
U.closeQuiet(srvrSock);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
index 543907f..3fca7af 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
@@ -29,7 +29,7 @@ public class StripedExecutorTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override public void beforeTest() {
- stripedExecSvc = new StripedExecutor(3, "foo name", "pool name", new JavaLogger());
+ stripedExecSvc = new StripedExecutor(3, "foo name", "pool name", new JavaLogger(), (thread, t) -> {});
}
/** {@inheritDoc} */
[13/14] ignite git commit: IGNITE-8153 Nodes fail to connect each
other when SSL is enabled - Fixes #3773.
Posted by ak...@apache.org.
IGNITE-8153 Nodes fail to connect each other when SSL is enabled - Fixes #3773.
Signed-off-by: Valentin Kulichenko <va...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f4de6df7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f4de6df7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f4de6df7
Branch: refs/heads/ignite-8201
Commit: f4de6df71b256506ce36c1c4e16533bb063782a0
Parents: a96ac04
Author: mcherkasov <mc...@gridgain.com>
Authored: Tue Apr 10 17:23:29 2018 -0700
Committer: Valentin Kulichenko <va...@gmail.com>
Committed: Tue Apr 10 17:23:29 2018 -0700
----------------------------------------------------------------------
.../ignite/internal/util/nio/ssl/BlockingSslHandler.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4de6df7/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java
index 638106f..0099c46 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java
@@ -373,9 +373,10 @@ public class BlockingSslHandler {
* @throws GridNioException If failed to pass event to the next filter.
*/
private Status unwrapHandshake() throws SSLException, IgniteCheckedException {
- // Flip input buffer so we can read the collected data.
- readFromNet();
+ if(!inNetBuf.hasRemaining())
+ readFromNet();
+ // Flip input buffer so we can read the collected data.
inNetBuf.flip();
SSLEngineResult res = unwrap0();
@@ -399,7 +400,10 @@ public class BlockingSslHandler {
else if (res.getStatus() == BUFFER_UNDERFLOW) {
inNetBuf.compact();
- inNetBuf = expandBuffer(inNetBuf, inNetBuf.capacity() * 2);
+ if(inNetBuf.capacity() == inNetBuf.limit())
+ inNetBuf = expandBuffer(inNetBuf, inNetBuf.capacity() * 2);
+
+ readFromNet();
}
else
// prepare to be written again
[10/14] ignite git commit: IGNITE-8069 IgniteOutOfMemoryException
should be handled accordingly to provided failure handler
Posted by ak...@apache.org.
IGNITE-8069 IgniteOutOfMemoryException should be handled accordingly to provided failure handler
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9bb4ce8a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9bb4ce8a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9bb4ce8a
Branch: refs/heads/ignite-8201
Commit: 9bb4ce8ab9770967d39f0acac5cdc1dc4230abb4
Parents: c807ae9
Author: Aleksey Plekhanov <pl...@gmail.com>
Authored: Tue Apr 10 18:54:03 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Apr 10 18:54:03 2018 +0300
----------------------------------------------------------------------
.../pagemem/impl/PageMemoryNoStoreImpl.java | 17 ++-
.../persistence/pagemem/PageMemoryImpl.java | 17 ++-
.../failure/AbstractFailureHandlerTest.java | 74 ++++++++++
.../ignite/failure/IoomFailureHandlerTest.java | 144 +++++++++++++++++++
.../persistence/pagemem/PageMemoryImplTest.java | 9 ++
.../ignite/testsuites/IgniteBasicTestSuite.java | 2 +
6 files changed, 259 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
index 7424af6..d4b22a6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
@@ -28,6 +28,8 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.mem.DirectMemoryProvider;
import org.apache.ignite.internal.mem.DirectMemoryRegion;
import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
@@ -158,6 +160,9 @@ public class PageMemoryNoStoreImpl implements PageMemory {
/** */
private final boolean trackAcquiredPages;
+ /** Shared context. */
+ private final GridCacheSharedContext<?, ?> ctx;
+
/**
* @param log Logger.
* @param directMemoryProvider Memory allocator to use.
@@ -184,6 +189,7 @@ public class PageMemoryNoStoreImpl implements PageMemory {
this.trackAcquiredPages = trackAcquiredPages;
this.memMetrics = memMetrics;
this.dataRegionCfg = dataRegionCfg;
+ this.ctx = sharedCtx;
sysPageSize = pageSize + PAGE_OVERHEAD;
@@ -288,8 +294,8 @@ public class PageMemoryNoStoreImpl implements PageMemory {
}
}
- if (relPtr == INVALID_REL_PTR)
- throw new IgniteOutOfMemoryException("Out of memory in data region [" +
+ if (relPtr == INVALID_REL_PTR) {
+ IgniteOutOfMemoryException oom = new IgniteOutOfMemoryException("Out of memory in data region [" +
"name=" + dataRegionCfg.getName() +
", initSize=" + U.readableSize(dataRegionCfg.getInitialSize(), false) +
", maxSize=" + U.readableSize(dataRegionCfg.getMaxSize(), false) +
@@ -299,6 +305,13 @@ public class PageMemoryNoStoreImpl implements PageMemory {
" ^-- Enable eviction or expiration policies"
);
+ if (ctx != null)
+ ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom));
+
+ throw oom;
+ }
+
+
assert (relPtr & ~PageIdUtils.PAGE_IDX_MASK) == 0 : U.hexLong(relPtr & ~PageIdUtils.PAGE_IDX_MASK);
// Assign page ID according to flags and partition ID.
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 46fb7dd..4463224 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -40,6 +40,8 @@ import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.mem.DirectMemoryProvider;
import org.apache.ignite.internal.mem.DirectMemoryRegion;
@@ -543,7 +545,7 @@ public class PageMemoryImpl implements PageMemoryEx {
catch (IgniteOutOfMemoryException oom) {
DataRegionConfiguration dataRegionCfg = getDataRegionConfiguration();
- throw (IgniteOutOfMemoryException) new IgniteOutOfMemoryException("Out of memory in data region [" +
+ IgniteOutOfMemoryException e = new IgniteOutOfMemoryException("Out of memory in data region [" +
"name=" + dataRegionCfg.getName() +
", initSize=" + U.readableSize(dataRegionCfg.getInitialSize(), false) +
", maxSize=" + U.readableSize(dataRegionCfg.getMaxSize(), false) +
@@ -551,7 +553,13 @@ public class PageMemoryImpl implements PageMemoryEx {
" ^-- Increase maximum off-heap memory size (DataRegionConfiguration.maxSize)" + U.nl() +
" ^-- Enable Ignite persistence (DataRegionConfiguration.persistenceEnabled)" + U.nl() +
" ^-- Enable eviction or expiration policies"
- ).initCause(oom);
+ );
+
+ e.initCause(oom);
+
+ ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+
+ throw e;
}
finally {
seg.writeLock().unlock();
@@ -746,6 +754,11 @@ public class PageMemoryImpl implements PageMemoryEx {
return absPtr;
}
+ catch (IgniteOutOfMemoryException oom) {
+ ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom));
+
+ throw oom;
+ }
finally {
seg.writeLock().unlock();
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java
new file mode 100644
index 0000000..dc5f1f5
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Abstract failure handler test.
+ */
+public class AbstractFailureHandlerTest extends GridCommonAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected FailureHandler getFailureHandler(String igniteInstanceName) {
+ return new DummyFailureHandler();
+ }
+
+ /**
+ * Gets dummy failure handler for ignite instance.
+ *
+ * @param ignite Ignite.
+ */
+ protected static DummyFailureHandler dummyFailureHandler(Ignite ignite) {
+ return (DummyFailureHandler)ignite.configuration().getFailureHandler();
+ }
+
+ /**
+ *
+ */
+ protected static class DummyFailureHandler implements FailureHandler {
+ /** Failure. */
+ private volatile boolean failure;
+
+ /** Failure context. */
+ private volatile FailureContext ctx;
+
+ /** {@inheritDoc} */
+ @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+ failure = true;
+
+ ctx = failureCtx;
+
+ return true;
+ }
+
+ /**
+ * @return Failure.
+ */
+ public boolean failure() {
+ return failure;
+ }
+
+ /**
+ * @return Failure context.
+ */
+ public FailureContext failureContext() {
+ return ctx;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java
new file mode 100644
index 0000000..a777f81
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.transactions.Transaction;
+
+/**
+ * IgniteOutOfMemoryError failure handler test.
+ */
+public class IoomFailureHandlerTest extends AbstractFailureHandlerTest {
+ /** Offheap size for memory policy. */
+ private static final int SIZE = 10 * 1024 * 1024;
+
+ /** Page size. */
+ static final int PAGE_SIZE = 2048;
+
+ /** Number of entries. */
+ static final int ENTRIES = 10_000;
+
+ /** PDS enabled. */
+ private boolean pds;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ DataStorageConfiguration dsCfg = new DataStorageConfiguration();
+
+ DataRegionConfiguration dfltPlcCfg = new DataRegionConfiguration();
+
+ dfltPlcCfg.setName("dfltPlc");
+ dfltPlcCfg.setInitialSize(SIZE);
+ dfltPlcCfg.setMaxSize(SIZE);
+
+ if (pds)
+ dfltPlcCfg.setPersistenceEnabled(true);
+
+ dsCfg.setDefaultDataRegionConfiguration(dfltPlcCfg);
+ dsCfg.setPageSize(PAGE_SIZE);
+
+ cfg.setDataStorageConfiguration(dsCfg);
+
+ CacheConfiguration<?, ?> ccfg = new CacheConfiguration<>()
+ .setName(DEFAULT_CACHE_NAME)
+ .setCacheMode(CacheMode.PARTITIONED)
+ .setBackups(0)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ cleanPersistenceDir();
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * Test IgniteOutOfMemoryException handling with no store.
+ */
+ public void testIoomErrorNoStoreHandling() throws Exception {
+ testIoomErrorHandling(false);
+ }
+
+ /**
+ * Test IgniteOutOfMemoryException handling with PDS.
+ */
+ public void testIoomErrorPdsHandling() throws Exception {
+ testIoomErrorHandling(true);
+ }
+
+ /**
+ * Test IOOME handling.
+ */
+ public void testIoomErrorHandling(boolean pds) throws Exception {
+ this.pds = pds;
+
+ IgniteEx ignite0 = startGrid(0);
+ IgniteEx ignite1 = startGrid(1);
+
+ try {
+ if (pds)
+ ignite0.cluster().active(true);
+
+ IgniteCache<Integer, Object> cache0 = ignite0.getOrCreateCache(DEFAULT_CACHE_NAME);
+ IgniteCache<Integer, Object> cache1 = ignite1.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+ awaitPartitionMapExchange();
+
+ try (Transaction tx = ignite0.transactions().txStart()) {
+ for (Integer i : primaryKeys(cache1, ENTRIES))
+ cache0.put(i, new byte[PAGE_SIZE / 3 * 2]);
+
+ tx.commit();
+ }
+ catch (Throwable ignore) {
+ // Expected.
+ }
+
+ assertFalse(dummyFailureHandler(ignite0).failure());
+ assertTrue(dummyFailureHandler(ignite1).failure());
+ assertTrue(X.hasCause(dummyFailureHandler(ignite1).failureContext().error(), IgniteOutOfMemoryException.class));
+ }
+ finally {
+ stopGrid(1);
+ stopGrid(0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
index 31af118..3697c4c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
@@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.NoOpFailureHandler;
import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
import org.apache.ignite.internal.mem.DirectMemoryProvider;
import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
@@ -41,6 +42,7 @@ import org.apache.ignite.internal.processors.cache.persistence.CheckpointWritePr
import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor;
import org.apache.ignite.internal.util.lang.GridInClosure3X;
import org.apache.ignite.plugin.PluginProvider;
@@ -268,10 +270,17 @@ public class PageMemoryImplTest extends GridCommonAbstractTest {
IgniteConfiguration igniteCfg = new IgniteConfiguration();
igniteCfg.setDataStorageConfiguration(new DataStorageConfiguration());
+ igniteCfg.setFailureHandler(new NoOpFailureHandler());
GridTestKernalContext kernalCtx = new GridTestKernalContext(new GridTestLog4jLogger(), igniteCfg);
kernalCtx.add(new IgnitePluginProcessor(kernalCtx, igniteCfg, Collections.<PluginProvider>emptyList()));
+ FailureProcessor failureProc = new FailureProcessor(kernalCtx);
+
+ failureProc.start();
+
+ kernalCtx.add(failureProc);
+
GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
kernalCtx,
null,
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index dd9cdfd..c4b7d92 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -21,6 +21,7 @@ import java.util.Set;
import junit.framework.TestSuite;
import org.apache.ignite.GridSuppressedExceptionSelfTest;
import org.apache.ignite.failure.FailureHandlerTriggeredTest;
+import org.apache.ignite.failure.IoomFailureHandlerTest;
import org.apache.ignite.failure.StopNodeFailureHandlerTest;
import org.apache.ignite.failure.StopNodeOrHaltFailureHandlerTest;
import org.apache.ignite.internal.ClassSetTest;
@@ -197,6 +198,7 @@ public class IgniteBasicTestSuite extends TestSuite {
suite.addTestSuite(FailureHandlerTriggeredTest.class);
suite.addTestSuite(StopNodeFailureHandlerTest.class);
suite.addTestSuite(StopNodeOrHaltFailureHandlerTest.class);
+ suite.addTestSuite(IoomFailureHandlerTest.class);
return suite;
}