You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ak...@apache.org on 2018/01/17 12:23:10 UTC
[12/43] ignite git commit: IGNITE-7203 Java 8 by default
http://git-wip-us.apache.org/repos/asf/ignite/blob/43a62cdb/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/math/IgniteSparseLocalMatrixMulBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/math/IgniteSparseLocalMatrixMulBenchmark.java b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/math/IgniteSparseLocalMatrixMulBenchmark.java
deleted file mode 100644
index de2c99b..0000000
--- a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/math/IgniteSparseLocalMatrixMulBenchmark.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.math;
-
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.impls.matrix.SparseLocalOnHeapMatrix;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteSparseLocalMatrixMulBenchmark extends IgniteAbstractMatrixMulBenchmark {
- /** {@inheritDoc} */
- @Override Matrix newMatrix(int rowSize, int colSize) {
- return new SparseLocalOnHeapMatrix(rowSize, colSize);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/43a62cdb/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/math/package-info.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/math/package-info.java b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/math/package-info.java
deleted file mode 100644
index d37e967..0000000
--- a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/math/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * ML Grid math benchmarks.
- */
-package org.apache.ignite.yardstick.ml.math;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/43a62cdb/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/package-info.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/package-info.java b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/package-info.java
deleted file mode 100644
index 4751f35..0000000
--- a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * ML Grid benchmarks.
- */
-package org.apache.ignite.yardstick.ml;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/43a62cdb/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/regression/IgniteOLSMultipleLinearRegressionBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/regression/IgniteOLSMultipleLinearRegressionBenchmark.java b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/regression/IgniteOLSMultipleLinearRegressionBenchmark.java
deleted file mode 100644
index 89b5471..0000000
--- a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/regression/IgniteOLSMultipleLinearRegressionBenchmark.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.regression;
-
-import java.util.Map;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionQRTrainer;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteOLSMultipleLinearRegressionBenchmark extends IgniteAbstractBenchmark {
- /** {@inheritDoc} */
- @Override public boolean test(Map<Object, Object> ctx) throws Exception {
- runLongly();
-
- return true;
- }
-
- /**
- * Based on OLSMultipleLinearRegressionTest#testLongly.
- */
- private void runLongly() {
- // Y values are first, then independent vars
- // Each row is one observation
- double[][] data = new double[][] {
- { 60323, 83.0, 234289, 2356, 1590, 107608, 1947 },
- { 61122, 88.5, 259426, 2325, 1456, 108632, 1948 },
- { 60171, 88.2, 258054, 3682, 1616, 109773, 1949 },
- { 61187, 89.5, 284599, 3351, 1650, 110929, 1950 },
- { 63221, 96.2, 328975, 2099, 3099, 112075, 1951 },
- { 63639, 98.1, 346999, 1932, 3594, 113270, 1952 },
- { 64989, 99.0, 365385, 1870, 3547, 115094, 1953 },
- { 63761, 100.0, 363112, 3578, 3350, 116219, 1954 },
- { 66019, 101.2, 397469, 2904, 3048, 117388, 1955 },
- { 67857, 104.6, 419180, 2822, 2857, 118734, 1956 },
- { 68169, 108.4, 442769, 2936, 2798, 120445, 1957 },
- { 66513, 110.8, 444546, 4681, 2637, 121950, 1958 },
- { 68655, 112.6, 482704, 3813, 2552, 123366, 1959 },
- { 69564, 114.2, 502601, 3931, 2514, 125368, 1960 },
- { 69331, 115.7, 518173, 4806, 2572, 127852, 1961 },
- { 70551, 116.9, 554894, 4007, 2827, 130081, 1962 }
- };
-
- final int nobs = 16;
- final int nvars = 6;
-
- LinearRegressionQRTrainer trainer = new LinearRegressionQRTrainer();
- LinearRegressionModel model = trainer.train(new DenseLocalOnHeapMatrix(data));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/43a62cdb/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/regression/package-info.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/regression/package-info.java b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/regression/package-info.java
deleted file mode 100644
index 0a5dc1a..0000000
--- a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/regression/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * ML Grid regression benchmarks.
- */
-package org.apache.ignite.yardstick.ml.regression;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/43a62cdb/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
deleted file mode 100644
index f8a7c08..0000000
--- a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.trees;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.thread.IgniteThread;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteColumnDecisionTreeGiniBenchmark extends IgniteAbstractBenchmark {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public boolean test(Map<Object, Object> ctx) throws Exception {
- // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
- // because we create ignite cache internally.
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- this.getClass().getSimpleName(), new Runnable() {
- /** {@inheritDoc} */
- @Override public void run() {
- // IMPL NOTE originally taken from ColumnDecisionTreeTrainerTest#testCacheMixedGini
- int totalPts = 1 << 10;
- int featCnt = 2;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
- catsInfo.put(1, 3);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1)).
- split(0, 1, new int[] {0, 2}).
- split(1, 0, -10.0);
-
- gen.testByGen(totalPts, ContinuousSplitCalculators.GINI.apply(ignite),
- RegionCalculators.GINI, RegionCalculators.MEAN, ignite);
- }
- });
-
- igniteThread.start();
-
- igniteThread.join();
-
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/43a62cdb/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
deleted file mode 100644
index f9d417f..0000000
--- a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.trees;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.thread.IgniteThread;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteColumnDecisionTreeVarianceBenchmark extends IgniteAbstractBenchmark {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public boolean test(Map<Object, Object> ctx) throws Exception {
- // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
- // because we create ignite cache internally.
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- this.getClass().getSimpleName(), new Runnable() {
- /** {@inheritDoc} */
- @Override public void run() {
- // IMPL NOTE originally taken from ColumnDecisionTreeTrainerTest#testCacheMixed
- int totalPts = 1 << 10;
- int featCnt = 2;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
- catsInfo.put(1, 3);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen
- = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1)).
- split(0, 1, new int[] {0, 2}).
- split(1, 0, -10.0);
-
- gen.testByGen(totalPts,
- ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, RegionCalculators.MEAN, ignite);
- }
- });
-
- igniteThread.start();
-
- igniteThread.join();
-
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/43a62cdb/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
deleted file mode 100644
index f9117f4..0000000
--- a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
+++ /dev/null
@@ -1,426 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.trees;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledVectorDouble;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.MatrixColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.util.Utils;
-
-/** */
-class SplitDataGenerator<V extends Vector> {
- /** */
- private static final Random rnd = new Random(12349L);
-
- /** */
- private static final double DELTA = 100.0;
-
- /** Map of the form of (is categorical -> list of region indexes). */
- private final Map<Boolean, List<Integer>> di;
-
- /** List of regions. */
- private final List<Region> regs;
-
- /** Data of bounds of regions. */
- private final Map<Integer, IgniteBiTuple<Double, Double>> boundsData;
-
- /** */
- private final Map<Integer, Integer> catFeaturesInfo;
-
- /** Supplier of vectors. */
- private final Supplier<V> supplier;
-
- /** Features count. */
- private final int featCnt;
-
- /**
- * Create SplitDataGenerator.
- *
- * @param featCnt Features count.
- * @param catFeaturesInfo Information about categorical features in form of map (feature index -> categories
- * count).
- * @param supplier Supplier of vectors.
- */
- SplitDataGenerator(int featCnt, Map<Integer, Integer> catFeaturesInfo, Supplier<V> supplier) {
- regs = new LinkedList<>();
- boundsData = new HashMap<>();
- this.supplier = supplier;
- this.featCnt = featCnt;
- this.catFeaturesInfo = catFeaturesInfo;
-
- // Divide indexes into indexes of categorical coordinates and indexes of continuous coordinates.
- di = IntStream.range(0, featCnt).
- boxed().
- collect(Collectors.partitioningBy(catFeaturesInfo::containsKey));
-
- // Categorical coordinates info.
- Map<Integer, CatCoordInfo> catCoords = new HashMap<>();
- di.get(true).forEach(i -> {
- BitSet bs = new BitSet();
- bs.set(0, catFeaturesInfo.get(i));
- catCoords.put(i, new CatCoordInfo(bs));
- });
-
- // Continuous coordinates info.
- Map<Integer, ContCoordInfo> contCoords = new HashMap<>();
- di.get(false).forEach(i -> {
- contCoords.put(i, new ContCoordInfo());
- boundsData.put(i, new IgniteBiTuple<>(-1.0, 1.0));
- });
-
- Region firstReg = new Region(catCoords, contCoords, 0);
- regs.add(firstReg);
- }
-
- /** */
- <D extends ContinuousRegionInfo> void testByGen(int totalPts,
- IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> calc,
- IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> catImpCalc,
- IgniteFunction<DoubleStream, Double> regCalc, Ignite ignite) {
-
- List<IgniteBiTuple<Integer, V>> lst = points(totalPts, (i, rn) -> i).collect(Collectors.toList());
-
- Collections.shuffle(lst, rnd);
-
- SparseDistributedMatrix m = new SparseDistributedMatrix(totalPts,
- featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
- Map<Integer, List<LabeledVectorDouble>> byRegion = new HashMap<>();
-
- int i = 0;
- for (IgniteBiTuple<Integer, V> bt : lst) {
- byRegion.putIfAbsent(bt.get1(), new LinkedList<>());
- byRegion.get(bt.get1()).add(asLabeledVector(bt.get2().getStorage().data()));
- m.setRow(i, bt.get2().getStorage().data());
- i++;
- }
-
- ColumnDecisionTreeTrainer<D> trainer =
- new ColumnDecisionTreeTrainer<>(3, calc, catImpCalc, regCalc, ignite);
-
- DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, catFeaturesInfo));
-
- byRegion.keySet().forEach(k -> mdl.apply(byRegion.get(k).get(0).features()));
- }
-
- /**
- * Split region by continuous coordinate using given threshold.
- *
- * @param regIdx Region index.
- * @param coordIdx Coordinate index.
- * @param threshold Threshold.
- * @return {@code this}.
- */
- SplitDataGenerator<V> split(int regIdx, int coordIdx, double threshold) {
- Region regToSplit = regs.get(regIdx);
- ContCoordInfo cci = regToSplit.contCoords.get(coordIdx);
-
- double left = cci.left;
- double right = cci.right;
-
- if (threshold < left || threshold > right)
- throw new MathIllegalArgumentException("Threshold is out of region bounds.");
-
- regToSplit.incTwoPow();
-
- Region newReg = Utils.copy(regToSplit);
- newReg.contCoords.get(coordIdx).left = threshold;
-
- regs.add(regIdx + 1, newReg);
- cci.right = threshold;
-
- IgniteBiTuple<Double, Double> bounds = boundsData.get(coordIdx);
- double min = bounds.get1();
- double max = bounds.get2();
- boundsData.put(coordIdx, new IgniteBiTuple<>(Math.min(threshold, min), Math.max(max, threshold)));
-
- return this;
- }
-
- /**
- * Split region by categorical coordinate.
- *
- * @param regIdx Region index.
- * @param coordIdx Coordinate index.
- * @param cats Categories allowed for the left sub region.
- * @return {@code this}.
- */
- SplitDataGenerator<V> split(int regIdx, int coordIdx, int[] cats) {
- BitSet subset = new BitSet();
- Arrays.stream(cats).forEach(subset::set);
- Region regToSplit = regs.get(regIdx);
- CatCoordInfo cci = regToSplit.catCoords.get(coordIdx);
-
- BitSet ssc = (BitSet)subset.clone();
- BitSet set = cci.bs;
- ssc.and(set);
- if (ssc.length() != subset.length())
- throw new MathIllegalArgumentException("Splitter set is not a subset of a parent subset.");
-
- ssc.xor(set);
- set.and(subset);
-
- regToSplit.incTwoPow();
- Region newReg = Utils.copy(regToSplit);
- newReg.catCoords.put(coordIdx, new CatCoordInfo(ssc));
-
- regs.add(regIdx + 1, newReg);
-
- return this;
- }
-
- /**
- * Get stream of points generated by this generator.
- *
- * @param ptsCnt Points count.
- */
- private Stream<IgniteBiTuple<Integer, V>> points(int ptsCnt, BiFunction<Double, Random, Double> f) {
- return IntStream.range(0, regs.size()).
- boxed().
- map(i -> regs.get(i).generatePoints(ptsCnt, f.apply((double)i, rnd), boundsData, di, supplier, rnd)
- .map(v -> new IgniteBiTuple<>(i, v))).flatMap(Function.identity());
- }
-
- /**
- * Convert double array to {@link LabeledVectorDouble}
- *
- * @param arr Array for conversion.
- * @return LabeledVectorDouble.
- */
- private static LabeledVectorDouble<DenseLocalOnHeapVector> asLabeledVector(double arr[]) {
- return new LabeledVectorDouble<>(new DenseLocalOnHeapVector(
- Arrays.copyOf(arr, arr.length - 1)), arr[arr.length - 1]);
- }
-
- /**
- * Categorical coordinate info.
- */
- private static class CatCoordInfo implements Serializable {
- /**
- * Defines categories which are included in this region
- */
- private final BitSet bs;
-
- /**
- * Construct CatCoordInfo.
- *
- * @param bs Bitset.
- */
- CatCoordInfo(BitSet bs) {
- this.bs = bs;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "CatCoordInfo [" +
- "bs=" + bs +
- ']';
- }
- }
-
- /**
- * Continuous coordinate info.
- */
- private static class ContCoordInfo implements Serializable {
- /**
- * Left (min) bound of region.
- */
- private double left;
-
- /**
- * Right (max) bound of region.
- */
- private double right;
-
- /**
- * Construct ContCoordInfo.
- */
- ContCoordInfo() {
- left = Double.NEGATIVE_INFINITY;
- right = Double.POSITIVE_INFINITY;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "ContCoordInfo [" +
- "left=" + left +
- ", right=" + right +
- ']';
- }
- }
-
- /**
- * Class representing information about region.
- */
- private static class Region implements Serializable {
- /**
- * Information about categorical coordinates restrictions of this region in form of
- * (coordinate index -> restriction)
- */
- private final Map<Integer, CatCoordInfo> catCoords;
-
- /**
- * Information about continuous coordinates restrictions of this region in form of
- * (coordinate index -> restriction)
- */
- private final Map<Integer, ContCoordInfo> contCoords;
-
- /**
- * Region should contain {@code 1/2^twoPow * totalPoints} points.
- */
- private int twoPow;
-
- /**
- * Construct region by information about restrictions on coordinates (features) values.
- *
- * @param catCoords Restrictions on categorical coordinates.
- * @param contCoords Restrictions on continuous coordinates
- * @param twoPow Region should contain {@code 1/2^twoPow * totalPoints} points.
- */
- Region(Map<Integer, CatCoordInfo> catCoords, Map<Integer, ContCoordInfo> contCoords, int twoPow) {
- this.catCoords = catCoords;
- this.contCoords = contCoords;
- this.twoPow = twoPow;
- }
-
- /** */
- int divideBy() {
- return 1 << twoPow;
- }
-
- /** */
- void incTwoPow() {
- twoPow++;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "Region [" +
- "catCoords=" + catCoords +
- ", contCoords=" + contCoords +
- ", twoPow=" + twoPow +
- ']';
- }
-
- /**
- * Generate continuous coordinate for this region.
- *
- * @param coordIdx Coordinate index.
- * @param boundsData Data with bounds
- * @param rnd Random numbers generator.
- * @return Categorical coordinate value.
- */
- double generateContCoord(int coordIdx, Map<Integer, IgniteBiTuple<Double, Double>> boundsData,
- Random rnd) {
- ContCoordInfo cci = contCoords.get(coordIdx);
- double left = cci.left;
- double right = cci.right;
-
- if (left == Double.NEGATIVE_INFINITY)
- left = boundsData.get(coordIdx).get1() - DELTA;
-
- if (right == Double.POSITIVE_INFINITY)
- right = boundsData.get(coordIdx).get2() + DELTA;
-
- double size = right - left;
-
- return left + rnd.nextDouble() * size;
- }
-
- /**
- * Generate categorical coordinate value for this region.
- *
- * @param coordIdx Coordinate index.
- * @param rnd Random numbers generator.
- * @return Categorical coordinate value.
- */
- double generateCatCoord(int coordIdx, Random rnd) {
- // Pick random bit.
- BitSet bs = catCoords.get(coordIdx).bs;
- int j = rnd.nextInt(bs.length());
-
- int i = 0;
- int bn = 0;
- int bnp = 0;
-
- while ((bn = bs.nextSetBit(bn)) != -1 && i <= j) {
- i++;
- bnp = bn;
- bn++;
- }
-
- return bnp;
- }
-
- /**
- * Generate points for this region.
- *
- * @param ptsCnt Count of points to generate.
- * @param val Label for all points in this region.
- * @param boundsData Data about bounds of continuous coordinates.
- * @param catCont Data about which categories can be in this region in the form (coordinate index -> list of
- * categories indexes).
- * @param s Vectors supplier.
- * @param rnd Random numbers generator.
- * @param <V> Type of vectors.
- * @return Stream of generated points for this region.
- */
- <V extends Vector> Stream<V> generatePoints(int ptsCnt, double val,
- Map<Integer, IgniteBiTuple<Double, Double>> boundsData, Map<Boolean, List<Integer>> catCont,
- Supplier<V> s,
- Random rnd) {
- return IntStream.range(0, ptsCnt / divideBy()).mapToObj(i -> {
- V v = s.get();
- int coordsCnt = v.size();
- catCont.get(false).forEach(ci -> v.setX(ci, generateContCoord(ci, boundsData, rnd)));
- catCont.get(true).forEach(ci -> v.setX(ci, generateCatCoord(ci, rnd)));
-
- v.setX(coordsCnt - 1, val);
- return v;
- });
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/43a62cdb/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/package-info.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/package-info.java b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/package-info.java
deleted file mode 100644
index fc379a6..0000000
--- a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * ML Grid decision tree benchmarks.
- */
-package org.apache.ignite.yardstick.ml.trees;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/43a62cdb/modules/yarn/pom.xml
----------------------------------------------------------------------
diff --git a/modules/yarn/pom.xml b/modules/yarn/pom.xml
index 28771ab..8f64890 100644
--- a/modules/yarn/pom.xml
+++ b/modules/yarn/pom.xml
@@ -61,17 +61,6 @@
<build>
<plugins>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.1</version>
- <configuration>
- <source>1.7</source>
- <target>1.7</target>
- </configuration>
- </plugin>
-
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4.1</version>
http://git-wip-us.apache.org/repos/asf/ignite/blob/43a62cdb/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index c42b6ef..c87b8bb 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -30,31 +30,27 @@
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
<version>16</version>
- <relativePath></relativePath>
</parent>
<properties>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+
<ignite.edition>fabric</ignite.edition>
- <hadoop.version>2.4.1</hadoop.version>
- <spark.hadoop.version>2.6.5</spark.hadoop.version>
- <spark.version>2.2.0</spark.version>
- <spring.version>4.3.7.RELEASE</spring.version> <!-- don't forget to update spring-data version -->
- <spring.data.version>1.13.1.RELEASE</spring.data.version> <!-- don't forget to update spring version -->
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.build.timestamp.format>MMMM d yyyy</maven.build.timestamp.format>
<doxygen.exec>doxygen</doxygen.exec>
<docfx.exec>docfx</docfx.exec>
<git.exec>git</git.exec>
- <maven.bundle.plugin.version>2.5.4</maven.bundle.plugin.version>
- <javadoc.opts>-XDenableSunApiLintControl</javadoc.opts>
+ <javadoc.opts>-Xdoclint:none</javadoc.opts>
<update.notifier.enabled.by.default>true</update.notifier.enabled.by.default>
- <!-- Dependency versions. -->
+ <!-- Dependency versions -->
<activemq.version>5.12.0</activemq.version>
<aopalliance.bundle.version>1.0_6</aopalliance.bundle.version>
<asm.version>4.2</asm.version>
- <aspectj.bundle.version>1.7.2_1</aspectj.bundle.version>
- <aspectj.version>1.7.2</aspectj.version>
+ <aspectj.bundle.version>1.8.13_1</aspectj.bundle.version>
+ <aspectj.version>1.8.13</aspectj.version>
<aws.sdk.bundle.version>1.10.12_1</aws.sdk.bundle.version>
<aws.sdk.version>1.11.75</aws.sdk.version>
<camel.version>2.16.0</camel.version>
@@ -74,14 +70,19 @@
<guava14.version>14.0.1</guava14.version>
<guava16.version>16.0.1</guava16.version>
<h2.version>1.4.195</h2.version>
+ <hadoop.version>2.4.1</hadoop.version>
+ <hamcrest.version>1.2</hamcrest.version>
<httpclient.version>4.5.1</httpclient.version>
<httpcore.version>4.4.3</httpcore.version>
<jackson.version>1.9.13</jackson.version>
<jackson2.version>2.6.5</jackson2.version>
+ <javassist.version>3.20.0-GA</javassist.version>
<javax.cache.bundle.version>1.0.0_1</javax.cache.bundle.version>
+ <javax.cache.tck.version>1.0.1</javax.cache.tck.version>
<javax.cache.version>1.0.0</javax.cache.version>
<jetbrains.annotations.version>13.0</jetbrains.annotations.version>
<jetty.version>9.2.11.v20150529</jetty.version>
+ <jmh.version>1.13</jmh.version>
<jms.spec.version>1.1.1</jms.spec.version>
<jsch.bundle.version>0.1.53_1</jsch.bundle.version>
<jsch.version>0.1.53</jsch.version>
@@ -92,30 +93,36 @@
<karaf.version>4.0.2</karaf.version>
<lucene.bundle.version>5.5.2_1</lucene.bundle.version>
<lucene.version>5.5.2</lucene.version>
+ <maven.bundle.plugin.version>2.5.4</maven.bundle.plugin.version>
+ <mysql.connector.version>5.1.39</mysql.connector.version>
+ <netlibjava.version>1.1.2</netlibjava.version>
<oro.bundle.version>2.0.8_6</oro.bundle.version>
<osgi.core.version>5.0.0</osgi.core.version>
<osgi.enterprise.version>5.0.0</osgi.enterprise.version>
<paho.version>1.0.2</paho.version>
+ <postgres.connector.version>9.4.1208.jre7</postgres.connector.version>
<rocketmq.version>4.2.0</rocketmq.version>
<scala210.jline.version>2.10.4</scala210.jline.version>
<scala210.library.version>2.10.6</scala210.library.version>
<scala211.library.version>2.11.8</scala211.library.version>
<slf4j.version>1.7.7</slf4j.version>
<slf4j16.version>1.6.4</slf4j16.version>
+ <spark.hadoop.version>2.6.5</spark.hadoop.version>
+ <spark.version>2.2.0</spark.version>
+ <spring.data.version>1.13.1.RELEASE</spring.data.version> <!-- don't forget to update spring version -->
+ <spring.version>4.3.7.RELEASE</spring.version> <!-- don't forget to update spring-data version -->
<spring41.osgi.feature.version>4.1.7.RELEASE_1</spring41.osgi.feature.version>
<tomcat.version>8.0.23</tomcat.version>
<twitter.hbc.version>2.2.0</twitter.hbc.version>
<yammer.metrics.annotation.version>2.2.0</yammer.metrics.annotation.version>
<yammer.metrics.core.version>2.2.0</yammer.metrics.core.version>
+ <yardstick.version>0.8.3</yardstick.version>
<zkclient.version>0.5</zkclient.version>
<zookeeper.version>3.4.6</zookeeper.version>
- <javassist.version>3.20.0-GA</javassist.version>
<!-- OSGI Manifest generation default property values -->
<osgi.import.package>*</osgi.import.package>
<osgi.export.package>{local-packages}</osgi.export.package>
- <osgi.private.package></osgi.private.package>
- <osgi.embed.dependency></osgi.embed.dependency>
<osgi.embed.transitive>false</osgi.embed.transitive>
<osgi.fail.ok>false</osgi.fail.ok>
@@ -570,15 +577,6 @@
<plugins>
<plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.1</version>
- <configuration>
- <source>1.7</source>
- <target>1.7</target>
- </configuration>
- </plugin>
-
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.17</version>
@@ -962,27 +960,6 @@
</profile>
<profile>
- <id>java8</id>
- <activation>
- <jdk>[1.8,)</jdk>
- </activation>
- <properties>
- <javadoc.opts>-Xdoclint:none</javadoc.opts>
- </properties>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.8</source>
- <target>1.8</target>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
-
- <profile>
<id>tools.jar-default</id>
<activation>
http://git-wip-us.apache.org/repos/asf/ignite/blob/43a62cdb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2c7379e..1d8637c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -96,6 +96,7 @@
<module>modules/zeromq</module>
<module>modules/rocketmq</module>
<module>modules/sqlline</module>
+ <module>modules/ml</module>
</modules>
<profiles>
@@ -130,7 +131,6 @@
<profile>
<id>all-other</id> <!-- used to update project versions and check all modules compilation -->
<modules> <!-- sorted alphabetically -->
- <module>modules/ml</module> <!-- it uses Java8 and can't be compiled by Java7 used to compile Ignite -->
<module>modules/web-console</module>
</modules>
</profile>
@@ -752,13 +752,6 @@
</plugins>
</build>
</profile>
-
- <profile>
- <id>ml</id>
- <modules>
- <module>modules/ml</module>
- </modules>
- </profile>
</profiles>
<build>