You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by za...@apache.org on 2019/09/08 12:51:16 UTC
[ignite] branch master updated: IGNITE-12024: [ML] PeerClassloading
for ml related lambdas (#6853)
This is an automated email from the ASF dual-hosted git repository.
zaleslaw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new cc7e40d IGNITE-12024: [ML] PeerClassloading for ml related lambdas (#6853)
cc7e40d is described below
commit cc7e40d94a4770ba75ab22174831c20f8e655275
Author: Alexey Platonov <40...@users.noreply.github.com>
AuthorDate: Sun Sep 8 15:51:07 2019 +0300
IGNITE-12024: [ML] PeerClassloading for ml related lambdas (#6853)
---
.../TrainingWithCustomPreprocessorsExample.java | 152 +++++++++++++
.../examples/ml/environment/package-info.java | 17 +-
modules/extdata/p2p/pom.xml | 6 +
.../ignite/tests/p2p/ml/CustomPreprocessor1.java} | 27 ++-
.../ignite/tests/p2p/ml/CustomPreprocessor2.java} | 27 ++-
.../ignite/tests/p2p/ml/CustomVectorizer.java} | 36 ++--
modules/ml/pom.xml | 6 +
.../apache/ignite/ml/clustering/gmm/GmmModel.java | 10 +-
.../ignite/ml/clustering/gmm/GmmTrainer.java | 5 +-
.../ignite/ml/clustering/kmeans/KMeansModel.java | 10 +-
.../ignite/ml/clustering/kmeans/KMeansTrainer.java | 5 +-
.../ignite/ml/composition/CompositionUtils.java | 2 +-
.../ml/composition/ModelOnFeaturesSubspace.java | 2 +-
.../ignite/ml/composition/ModelsComposition.java | 9 +-
.../ignite/ml/composition/bagging/BaggedModel.java | 10 +-
.../ml/composition/bagging/BaggedTrainer.java | 4 +-
.../boosting/GDBBinaryClassifierTrainer.java | 10 +-
.../ignite/ml/composition/boosting/GDBTrainer.java | 8 +-
.../boosting/convergence/ConvergenceChecker.java | 8 +-
.../parallel/ModelsParallelComposition.java | 2 +-
.../parallel/TrainersParallelComposition.java | 4 +-
.../sequential/ModelsSequentialComposition.java | 2 +-
.../sequential/TrainersSequentialComposition.java | 2 +-
.../MeanValuePredictionsAggregator.java | 2 +-
.../OnMajorityPredictionsAggregator.java | 2 +-
.../WeightedPredictionsAggregator.java | 2 +-
.../stacking/StackedDatasetTrainer.java | 4 +-
.../ml/composition/stacking/StackedModel.java | 2 +-
.../apache/ignite/ml/dataset/DatasetBuilder.java | 5 +-
.../apache/ignite/ml/dataset/DatasetFactory.java | 43 +++-
.../ml/dataset/feature/extractor/Vectorizer.java | 2 +-
.../extractor/impl/BinaryObjectVectorizer.java | 10 +-
.../extractor/impl/DoubleArrayVectorizer.java | 11 +-
.../feature/extractor/impl/DummyVectorizer.java | 9 +-
.../extractor/impl/LabeledDummyVectorizer.java | 14 +-
.../ml/dataset/impl/cache/CacheBasedDataset.java | 22 +-
.../impl/cache/CacheBasedDatasetBuilder.java | 38 +++-
.../ml/dataset/impl/cache/util/ComputeUtils.java | 66 +++++-
.../ml/dataset/impl/local/LocalDatasetBuilder.java | 4 +-
.../DefaultLearningEnvironmentBuilder.java | 9 +
.../ignite/ml/environment/LearningEnvironment.java | 18 ++
.../ml/environment/deploy/DeployableObject.java} | 22 +-
.../ml/environment/deploy/DeployingContext.java | 55 +++++
.../environment/deploy/DeployingContextImpl.java | 76 +++++++
.../ml/environment/deploy/package-info.java} | 16 +-
.../java/org/apache/ignite/ml/knn/KNNTrainer.java | 6 +-
.../ignite/ml/knn/NNClassificationModel.java | 9 +-
.../ignite/ml/knn/ann/ANNClassificationModel.java | 2 +-
.../ml/knn/ann/ANNClassificationTrainer.java | 5 +-
.../org/apache/ignite/ml/knn/utils/KNNUtils.java | 7 +-
.../ignite/ml/math/isolve/lsqr/LSQROnHeap.java | 7 +-
.../ignite/ml/multiclass/MultiClassModel.java | 2 +-
.../ignite/ml/multiclass/OneVsRestTrainer.java | 4 +-
.../compound/CompoundNaiveBayesModel.java | 10 +-
.../compound/CompoundNaiveBayesTrainer.java | 2 +-
.../discrete/DiscreteNaiveBayesModel.java | 11 +-
.../discrete/DiscreteNaiveBayesTrainer.java | 4 +-
.../gaussian/GaussianNaiveBayesModel.java | 11 +-
.../gaussian/GaussianNaiveBayesTrainer.java | 4 +-
.../java/org/apache/ignite/ml/nn/MLPTrainer.java | 5 +-
.../apache/ignite/ml/nn/MultilayerPerceptron.java | 2 +-
.../org/apache/ignite/ml/pipeline/Pipeline.java | 7 +-
.../org/apache/ignite/ml/pipeline/PipelineMdl.java | 10 +-
.../ml/preprocessing/PreprocessingTrainer.java | 15 ++
.../binarization/BinarizationPreprocessor.java | 10 +-
.../developer/MappedPreprocessor.java | 10 +-
.../developer/PatchedPreprocessor.java | 9 +-
.../encoding/EncoderPreprocessor.java | 4 +-
.../ml/preprocessing/encoding/EncoderTrainer.java | 2 +-
.../onehotencoder/OneHotEncoderPreprocessor.java | 10 +-
.../stringencoder/StringEncoderPreprocessor.java | 10 +-
.../imputing/ImputerPreprocessor.java | 10 +-
.../ml/preprocessing/imputing/ImputerTrainer.java | 2 +-
.../maxabsscaling/MaxAbsScalerPreprocessor.java | 10 +-
.../maxabsscaling/MaxAbsScalerTrainer.java | 2 +-
.../minmaxscaling/MinMaxScalerPreprocessor.java | 10 +-
.../minmaxscaling/MinMaxScalerTrainer.java | 2 +-
.../normalization/NormalizationPreprocessor.java | 10 +-
.../StandardScalerPreprocessor.java | 10 +-
.../standardscaling/StandardScalerTrainer.java | 2 +-
.../linear/LinearRegressionLSQRTrainer.java | 6 +-
.../regressions/linear/LinearRegressionModel.java | 2 +-
.../linear/LinearRegressionSGDTrainer.java | 2 +-
.../logistic/LogisticRegressionModel.java | 2 +-
.../logistic/LogisticRegressionSGDTrainer.java | 2 +-
.../apache/ignite/ml/sql/SqlDatasetBuilder.java | 2 +-
.../ml/svm/SVMLinearClassificationModel.java | 2 +-
.../ml/svm/SVMLinearClassificationTrainer.java | 5 +-
.../ml/trainers/AdaptableDatasetTrainer.java | 4 +-
.../apache/ignite/ml/trainers/DatasetTrainer.java | 43 +++-
.../org/apache/ignite/ml/tree/DecisionTree.java | 5 +-
.../ml/tree/DecisionTreeConditionalNode.java | 2 +-
.../ignite/ml/tree/DecisionTreeLeafNode.java | 2 +-
.../tree/boosting/GDBOnTreesLearningStrategy.java | 7 +-
.../ml/tree/randomforest/RandomForestTrainer.java | 5 +-
.../test/java/org/apache/ignite/ml/TestUtils.java | 2 +-
.../ignite/ml/composition/bagging/BaggingTest.java | 5 +-
.../mean/MeanAbsValueConvergenceCheckerTest.java | 6 +-
.../MedianOfMedianConvergenceCheckerTest.java | 3 +-
.../impl/cache/CacheBasedDatasetBuilderTest.java | 6 +-
.../dataset/impl/cache/CacheBasedDatasetTest.java | 6 +-
.../dataset/impl/cache/util/ComputeUtilsTest.java | 12 +-
.../impl/local/LocalDatasetBuilderTest.java | 3 +-
.../ml/environment/EnvironmentTestSuite.java | 6 +-
.../ml/environment/LearningEnvironmentTest.java | 5 +-
.../deploy/DeployingContextImplTest.java | 150 +++++++++++++
.../ml/environment/deploy/MLDeployingTest.java | 240 +++++++++++++++++++++
.../ignite/ml/math/isolve/lsqr/LSQROnHeapTest.java | 8 +-
.../DataStreamGeneratorFillCacheTest.java | 53 +++--
.../util/generators/DataStreamGeneratorTest.java | 3 +-
.../cluster/spec/TensorFlowServerAddressSpec.java | 1 -
111 files changed, 1368 insertions(+), 264 deletions(-)
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/environment/TrainingWithCustomPreprocessorsExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/environment/TrainingWithCustomPreprocessorsExample.java
new file mode 100644
index 0000000..19e1262
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/environment/TrainingWithCustomPreprocessorsExample.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.environment;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
+import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
+import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
+import org.apache.ignite.ml.pipeline.Pipeline;
+import org.apache.ignite.ml.pipeline.PipelineMdl;
+import org.apache.ignite.ml.preprocessing.PreprocessingTrainer;
+import org.apache.ignite.ml.preprocessing.Preprocessor;
+import org.apache.ignite.ml.preprocessing.imputing.ImputerTrainer;
+import org.apache.ignite.ml.preprocessing.minmaxscaling.MinMaxScalerTrainer;
+import org.apache.ignite.ml.preprocessing.normalization.NormalizationTrainer;
+import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
+import org.apache.ignite.ml.selection.scoring.metric.regression.RegressionMetricValues;
+import org.apache.ignite.ml.selection.scoring.metric.regression.RegressionMetrics;
+import org.apache.ignite.ml.structures.LabeledVector;
+import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
+import org.apache.ignite.ml.util.MLSandboxDatasets;
+import org.apache.ignite.ml.util.SandboxMLCache;
+
+/**
+ * This example demostrates an ability of using custom client classes in cluster in case of absence of
+ * these classes on server nodes. Preprocessors (see {@link Preprocessor}, preprocessor trainers
+ * (see {@link PreprocessingTrainer} and vectorizers (see {@link Vectorizer}) can be defined in client code and
+ * deployed to server nodes during training phase.
+ *
+ * For demonstrating of deployment abilities of ml-related classes you can run this example with binary build.
+ * NOTE: This binary build should be run with copied basic ml libs from optional directory (ignite-ml at least).
+ */
+public class TrainingWithCustomPreprocessorsExample {
+ /**
+ * Run example.
+ *
+ * @param args Command line arguments.
+ * @throws Exception Exception.
+ */
+ public static void main(String[] args) throws Exception {
+ try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+ IgniteCache<Integer, Vector> trainingSet = new SandboxMLCache(ignite)
+ .fillCacheWith(MLSandboxDatasets.BOSTON_HOUSE_PRICES);
+
+ Vectorizer<Integer, Vector, Integer, Double> basicVectorizer = new DummyVectorizer<Integer>()
+ .labeled(Vectorizer.LabelCoordinate.FIRST);
+
+ Preprocessor<Integer, Vector> imputingPreprocessor = new ImputerTrainer<Integer, Vector>()
+ .fit(ignite, trainingSet, basicVectorizer);
+
+ // In-place definition of custom preprocessor by lambda expression.
+ Preprocessor<Integer, Vector> customPreprocessor = (k, v) -> {
+ LabeledVector res = imputingPreprocessor.apply(k, v);
+ double fifthFeature = res.features().get(5);
+
+ Vector updatedVector = res.features().set(5, fifthFeature > 0 ? Math.log(fifthFeature) : -1);
+ return updatedVector.labeled(res.label());
+ };
+
+ Vectorizer9000 customVectorizer = new Vectorizer9000(customPreprocessor);
+
+ PipelineMdl<Integer, Vector> mdl = new Pipeline<Integer, Vector, Integer, Double>()
+ .addVectorizer(customVectorizer)
+ .addPreprocessingTrainer(new MinMaxScalerTrainer<Integer, Vector>())
+ .addPreprocessingTrainer(new NormalizationTrainer<Integer, Vector>().withP(1))
+ .addPreprocessingTrainer(getCustomTrainer())
+ .addTrainer(new DecisionTreeClassificationTrainer(5, 0))
+ .fit(ignite, trainingSet);
+
+ System.out.println(">>> Perform scoring.");
+ double score = Evaluator.evaluate(
+ trainingSet,
+ mdl,
+ mdl.getPreprocessor(),
+ new RegressionMetrics().withMetric(RegressionMetricValues::r2)
+ );
+
+ System.out.println(">>> R^2 score: " + score);
+ }
+ }
+
+ /**
+ * Custom trainer for preprocessor. This trainer just returns custom preprocessor based on given
+ * preprocessor. Such trainers could be used for injecting custom preprocessors to Pipelines.
+ *
+ * @return Trainer for preprocessor that normalizes given vectors.
+ */
+ private static PreprocessingTrainer<Integer, Vector> getCustomTrainer() {
+ return new PreprocessingTrainer<Integer, Vector>() {
+ @Override public Preprocessor<Integer, Vector> fit(LearningEnvironmentBuilder envBuilder,
+ DatasetBuilder<Integer, Vector> datasetBuilder,
+ Preprocessor<Integer, Vector> basePreprocessor) {
+ return (k, v) -> {
+ LabeledVector res = basePreprocessor.apply(k, v);
+ return res.features().normalize().labeled(res.label());
+ };
+ }
+ };
+ }
+
+ /**
+ * Custom vectorizer based on preprocessor. This vectorizer extends given vectors by values equal to
+ * log(feature) for each feature in vector.
+ */
+ private static class Vectorizer9000 extends Vectorizer.VectorizerAdapter<Integer, Vector, Integer, Double> {
+ /** Base preprocessor. */
+ private final Preprocessor<Integer, Vector> basePreprocessor;
+
+ /**
+ * Creates an instance of vectorizer.
+ *
+ * @param basePreprocessor Base preprocessor.
+ */
+ public Vectorizer9000(Preprocessor<Integer, Vector> basePreprocessor) {
+ this.basePreprocessor = basePreprocessor;
+ }
+
+ /** {@inheritDoc} */
+ @Override public LabeledVector<Double> apply(Integer key, Vector value) {
+ LabeledVector<Double> baseVec = basePreprocessor.apply(key, value);
+ double[] features = new double[baseVec.size() * 2];
+ for(int i = 0; i < baseVec.size(); i++) {
+ features[i] = baseVec.get(i);
+ double logValue = Math.log(baseVec.get(i));
+ features[i + baseVec.size()] = Double.isInfinite(logValue) || Double.isNaN(logValue) ? -1. : logValue;
+ }
+
+ LabeledVector<Double> res = VectorUtils.of(features).labeled(baseVec.label());
+ return res;
+ }
+ }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/environment/EnvironmentTestSuite.java b/examples/src/main/java/org/apache/ignite/examples/ml/environment/package-info.java
similarity index 68%
copy from modules/ml/src/test/java/org/apache/ignite/ml/environment/EnvironmentTestSuite.java
copy to examples/src/main/java/org/apache/ignite/examples/ml/environment/package-info.java
index ae9f490..f15cac4 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/environment/EnvironmentTestSuite.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/environment/package-info.java
@@ -15,19 +15,8 @@
* limitations under the License.
*/
-package org.apache.ignite.ml.environment;
-
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-
/**
- * Test suite for all tests located in org.apache.ignite.ml.trees package.
+ * <!-- Package description. -->
+ * This package contains examples of learning environment features.
*/
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
- LearningEnvironmentBuilderTest.class,
- LearningEnvironmentTest.class,
- PromiseTest.class
-})
-public class EnvironmentTestSuite {
-}
+package org.apache.ignite.examples.ml.environment;
diff --git a/modules/extdata/p2p/pom.xml b/modules/extdata/p2p/pom.xml
index 9c073d8..981c07c 100644
--- a/modules/extdata/p2p/pom.xml
+++ b/modules/extdata/p2p/pom.xml
@@ -41,6 +41,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-ml</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/MeanValuePredictionsAggregator.java b/modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/ml/CustomPreprocessor1.java
similarity index 52%
copy from modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/MeanValuePredictionsAggregator.java
copy to modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/ml/CustomPreprocessor1.java
index 01e693d..2015acd 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/MeanValuePredictionsAggregator.java
+++ b/modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/ml/CustomPreprocessor1.java
@@ -15,18 +15,27 @@
* limitations under the License.
*/
-package org.apache.ignite.ml.composition.predictionsaggregator;
+package org.apache.ignite.tests.p2p.ml;
-import java.util.Arrays;
-import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+import org.apache.ignite.ml.preprocessing.Preprocessor;
+import org.apache.ignite.ml.structures.LabeledVector;
/**
- * Predictions aggregator returning the mean value of predictions.
+ *
*/
-public class MeanValuePredictionsAggregator implements PredictionsAggregator {
- /** {@inheritDoc} */
- @Override public Double apply(double[] estimations) {
- A.notEmpty(estimations, "estimations vector");
- return Arrays.stream(estimations).reduce(0.0, Double::sum) / estimations.length;
+public class CustomPreprocessor1 implements Preprocessor<Integer, Vector> {
+ /** */
+ private final Preprocessor<Integer, Vector> basePreprocessor;
+
+ /** */
+ public CustomPreprocessor1(Preprocessor<Integer, Vector> basePreprocessor) {
+ this.basePreprocessor = basePreprocessor;
+ }
+
+ /** */
+ @Override public LabeledVector apply(Integer integer, Vector vector) {
+ LabeledVector res = basePreprocessor.apply(integer, vector);
+ return res.features().normalize().labeled(res.label());
}
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/MeanValuePredictionsAggregator.java b/modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/ml/CustomPreprocessor2.java
similarity index 52%
copy from modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/MeanValuePredictionsAggregator.java
copy to modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/ml/CustomPreprocessor2.java
index 01e693d..fc2d424 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/MeanValuePredictionsAggregator.java
+++ b/modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/ml/CustomPreprocessor2.java
@@ -15,18 +15,27 @@
* limitations under the License.
*/
-package org.apache.ignite.ml.composition.predictionsaggregator;
+package org.apache.ignite.tests.p2p.ml;
-import java.util.Arrays;
-import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+import org.apache.ignite.ml.preprocessing.Preprocessor;
+import org.apache.ignite.ml.structures.LabeledVector;
/**
- * Predictions aggregator returning the mean value of predictions.
+ *
*/
-public class MeanValuePredictionsAggregator implements PredictionsAggregator {
- /** {@inheritDoc} */
- @Override public Double apply(double[] estimations) {
- A.notEmpty(estimations, "estimations vector");
- return Arrays.stream(estimations).reduce(0.0, Double::sum) / estimations.length;
+public class CustomPreprocessor2 implements Preprocessor<Integer, Vector> {
+ /** */
+ private final Preprocessor<Integer, Vector> basePreprocessor;
+
+ /** */
+ public CustomPreprocessor2(Preprocessor<Integer, Vector> basePreprocessor) {
+ this.basePreprocessor = basePreprocessor;
+ }
+
+ /** */
+ @Override public LabeledVector apply(Integer integer, Vector vector) {
+ LabeledVector res = basePreprocessor.apply(integer, vector);
+ return res.features().times(2.0).labeled(res.label());
}
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/DummyVectorizer.java b/modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/ml/CustomVectorizer.java
similarity index 55%
copy from modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/DummyVectorizer.java
copy to modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/ml/CustomVectorizer.java
index 8cde8d1..14d408c 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/DummyVectorizer.java
+++ b/modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/ml/CustomVectorizer.java
@@ -15,38 +15,36 @@
* limitations under the License.
*/
-package org.apache.ignite.ml.dataset.feature.extractor.impl;
+package org.apache.ignite.tests.p2p.ml;
import java.io.Serializable;
-import org.apache.ignite.ml.dataset.feature.extractor.ExtractionUtils;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.math.primitives.vector.Vector;
/**
- * Vectorizer on Vector.
*
- * @param <K> Type of key.
*/
-public class DummyVectorizer<K> extends ExtractionUtils.ArrayLikeVectorizer<K, Vector> {
- /** Serial version uid. */
- private static final long serialVersionUID = -6225354615212148224L;
-
- /**
- * Creates an instance of Vectorizer.
- *
- * @param coords Coordinates.
- */
- public DummyVectorizer(Integer ... coords) {
- super(coords);
+public class CustomVectorizer extends Vectorizer<Integer, Vector, Integer, Double> {
+ /** {@inheritDoc} */
+ @Override protected Serializable feature(Integer coord, Integer key, Vector value) {
+ return value.get(coord);
}
/** {@inheritDoc} */
- @Override protected Serializable feature(Integer coord, K key, Vector value) {
- return value.getRaw(coord);
+ @Override protected Double label(Integer coord, Integer key, Vector value) {
+ return value.get(coord);
}
/** {@inheritDoc} */
- @Override protected int sizeOf(K key, Vector value) {
- return value.size();
+ @Override protected Double zero() {
+ return 0.0;
}
+ /** {@inheritDoc} */
+ @Override protected List<Integer> allCoords(Integer key, Vector value) {
+ return IntStream.range(0, value.size()).boxed().collect(Collectors.toList());
+ }
}
diff --git a/modules/ml/pom.xml b/modules/ml/pom.xml
index bb66780..c95ab26 100644
--- a/modules/ml/pom.xml
+++ b/modules/ml/pom.xml
@@ -141,6 +141,12 @@
<artifactId>commons-math3</artifactId>
<version>${commons.math.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.7</version>
+ </dependency>
</dependencies>
<profiles>
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/gmm/GmmModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/gmm/GmmModel.java
index b2a526e..c4a765f 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/gmm/GmmModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/gmm/GmmModel.java
@@ -17,8 +17,10 @@
package org.apache.ignite.ml.clustering.gmm;
+import java.util.Collections;
import java.util.List;
import org.apache.ignite.ml.IgniteModel;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.stat.DistributionMixture;
import org.apache.ignite.ml.math.stat.MultivariateGaussianDistribution;
@@ -30,7 +32,8 @@ import org.apache.ignite.ml.math.stat.MultivariateGaussianDistribution;
* compute likelihood vector where each component of it is a probability of cluster of mixture (see {@link
* #likelihood(Vector)}).
*/
-public class GmmModel extends DistributionMixture<MultivariateGaussianDistribution> implements IgniteModel<Vector, Double> {
+public class GmmModel extends DistributionMixture<MultivariateGaussianDistribution> implements IgniteModel<Vector, Double>,
+ DeployableObject {
/** Serial version uid. */
private static final long serialVersionUID = -4484174539118240037L;
@@ -48,4 +51,9 @@ public class GmmModel extends DistributionMixture<MultivariateGaussianDistributi
@Override public Double predict(Vector input) {
return (double)likelihood(input).maxElement().index();
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.emptyList();
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/gmm/GmmTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/gmm/GmmTrainer.java
index d6cce8e..591cf26 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/gmm/GmmTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/gmm/GmmTrainer.java
@@ -103,7 +103,7 @@ public class GmmTrainer extends SingleLabelDatasetTrainer<GmmModel> {
}
/** {@inheritDoc} */
- @Override public <K, V> GmmModel fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> GmmModel fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> extractor) {
return updateModel(null, datasetBuilder, extractor);
}
@@ -482,7 +482,8 @@ public class GmmTrainer extends SingleLabelDatasetTrainer<GmmModel> {
try (Dataset<EmptyContext, GmmPartitionData> dataset = datasetBuilder.build(envBuilder,
new EmptyContextBuilder<>(),
- new GmmPartitionData.Builder<>(extractor, maxCountOfClusters)
+ new GmmPartitionData.Builder<>(extractor, maxCountOfClusters),
+ learningEnvironment()
)) {
if (mdl != null) {
if (initialMeans != null)
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModel.java
index 500c5bf..f1f677f 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModel.java
@@ -18,10 +18,12 @@
package org.apache.ignite.ml.clustering.kmeans;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.ignite.ml.Exportable;
import org.apache.ignite.ml.Exporter;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.Tracer;
import org.apache.ignite.ml.math.distances.DistanceMeasure;
import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -30,7 +32,8 @@ import org.apache.ignite.ml.util.ModelTrace;
/**
* This class encapsulates result of clusterization by KMeans algorithm.
*/
-public class KMeansModel implements ClusterizationModel<Vector, Integer>, Exportable<KMeansModelFormat> {
+public final class KMeansModel implements ClusterizationModel<Vector, Integer>, Exportable<KMeansModelFormat>,
+ DeployableObject {
/** Centers of clusters. */
private final Vector[] centers;
@@ -125,4 +128,9 @@ public class KMeansModel implements ClusterizationModel<Vector, Integer>, Export
.addField("centroids", centersList)
.toString();
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(distanceMeasure);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java
index 9f3483d..05f41b5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java
@@ -63,7 +63,7 @@ public class KMeansTrainer extends SingleLabelDatasetTrainer<KMeansModel> {
private DistanceMeasure distance = new EuclideanDistance();
/** {@inheritDoc} */
- @Override public <K, V> KMeansModel fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> KMeansModel fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> preprocessor) {
return updateModel(null, datasetBuilder, preprocessor);
}
@@ -86,7 +86,8 @@ public class KMeansTrainer extends SingleLabelDatasetTrainer<KMeansModel> {
try (Dataset<EmptyContext, LabeledVectorSet<LabeledVector>> dataset = datasetBuilder.build(
envBuilder,
(env, upstream, upstreamSize) -> new EmptyContext(),
- partDataBuilder
+ partDataBuilder,
+ learningEnvironment()
)) {
final Integer cols = dataset.compute(org.apache.ignite.ml.structures.Dataset::colSize, (a, b) -> {
if (a == null)
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/CompositionUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/CompositionUtils.java
index 665d9f0..a98de0d 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/CompositionUtils.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/CompositionUtils.java
@@ -44,7 +44,7 @@ public class CompositionUtils {
DatasetTrainer<? extends M, L> trainer) {
return new DatasetTrainer<IgniteModel<I, O>, L>() {
/** {@inheritDoc} */
- @Override public <K, V> IgniteModel<I, O> fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> IgniteModel<I, O> fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> extractor) {
return trainer.fit(datasetBuilder, extractor);
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelOnFeaturesSubspace.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelOnFeaturesSubspace.java
index e3c1a18..69292e6 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelOnFeaturesSubspace.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelOnFeaturesSubspace.java
@@ -28,7 +28,7 @@ import org.apache.ignite.ml.util.ModelTrace;
/**
* Model trained on a features subspace with mapping from original features space to subspace.
*/
-public class ModelOnFeaturesSubspace implements IgniteModel<Vector, Double> {
+public final class ModelOnFeaturesSubspace implements IgniteModel<Vector, Double> {
/**
* Features mapping to subspace.
*/
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelsComposition.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelsComposition.java
index eef5e62..3942b9e 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelsComposition.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelsComposition.java
@@ -23,13 +23,15 @@ import org.apache.ignite.ml.Exportable;
import org.apache.ignite.ml.Exporter;
import org.apache.ignite.ml.IgniteModel;
import org.apache.ignite.ml.composition.predictionsaggregator.PredictionsAggregator;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.util.ModelTrace;
/**
* Model consisting of several models and prediction aggregation strategy.
*/
-public class ModelsComposition implements IgniteModel<Vector, Double>, Exportable<ModelsCompositionFormat> {
+public class ModelsComposition implements IgniteModel<Vector, Double>, Exportable<ModelsCompositionFormat>,
+ DeployableObject {
/**
* Predictions aggregator.
*/
@@ -98,4 +100,9 @@ public class ModelsComposition implements IgniteModel<Vector, Double>, Exportabl
.addField("models", models)
.toString();
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(predictionsAggregator);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/bagging/BaggedModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/bagging/BaggedModel.java
index c59a634..f7b0e22 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/bagging/BaggedModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/bagging/BaggedModel.java
@@ -17,14 +17,17 @@
package org.apache.ignite.ml.composition.bagging;
+import java.util.Collections;
+import java.util.List;
import org.apache.ignite.ml.IgniteModel;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.primitives.vector.Vector;
/**
* This class represents model produced by {@link BaggedTrainer}.
* It is a wrapper around inner representation of model produced by {@link BaggedTrainer}.
*/
-public class BaggedModel implements IgniteModel<Vector, Double> {
+public final class BaggedModel implements IgniteModel<Vector, Double>, DeployableObject {
/** Inner representation of model produced by {@link BaggedTrainer}. */
private IgniteModel<Vector, Double> mdl;
@@ -54,4 +57,9 @@ public class BaggedModel implements IgniteModel<Vector, Double> {
@Override public void close() {
mdl.close();
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(mdl);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/bagging/BaggedTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/bagging/BaggedTrainer.java
index 5b2c6b2..9b0b3ec 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/bagging/BaggedTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/bagging/BaggedTrainer.java
@@ -148,7 +148,7 @@ public class BaggedTrainer<L> extends
/** {@inheritDoc} */
- @Override public <K, V> BaggedModel fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> BaggedModel fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> preprocessor) {
IgniteModel<Vector, Double> fit = getTrainer().fit(datasetBuilder, preprocessor);
return new BaggedModel(fit);
@@ -157,6 +157,8 @@ public class BaggedTrainer<L> extends
/** {@inheritDoc} */
@Override public <K, V> BaggedModel update(BaggedModel mdl, DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> preprocessor) {
+ learningEnvironment().initDeployingContext(preprocessor);
+
IgniteModel<Vector, Double> updated = getTrainer().update(mdl.model(), datasetBuilder, preprocessor);
return new BaggedModel(updated);
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBBinaryClassifierTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBBinaryClassifierTrainer.java
index cb2222a..577e29d 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBBinaryClassifierTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBBinaryClassifierTrainer.java
@@ -69,13 +69,15 @@ public abstract class GDBBinaryClassifierTrainer extends GDBTrainer {
/** {@inheritDoc} */
@Override protected <V, K> boolean learnLabels(DatasetBuilder<K, V> builder,
Preprocessor<K, V> preprocessor) {
+ learningEnvironment().initDeployingContext(preprocessor);
Set<Double> uniqLabels = builder.build(
envBuilder,
new EmptyContextBuilder<>(),
- new LabeledDatasetPartitionDataBuilderOnHeap<>(preprocessor))
- .compute((IgniteFunction<LabeledVectorSet<LabeledVector>, Set<Double>>)x ->
- Arrays.stream(x.labels()).boxed().collect(Collectors.toSet()), (a, b) -> {
+ new LabeledDatasetPartitionDataBuilderOnHeap<>(preprocessor),
+ learningEnvironment()
+ ).compute((IgniteFunction<LabeledVectorSet<LabeledVector>, Set<Double>>)x ->
+ Arrays.stream(x.labels()).boxed().collect(Collectors.toSet()), (a, b) -> {
if (a == null)
return b;
if (b == null)
@@ -83,7 +85,7 @@ public abstract class GDBBinaryClassifierTrainer extends GDBTrainer {
a.addAll(b);
return a;
}
- );
+ );
if (uniqLabels != null && uniqLabels.size() == 2) {
ArrayList<Double> lblsArr = new ArrayList<>(uniqLabels);
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBTrainer.java
index c57c700..ad35d80 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBTrainer.java
@@ -87,7 +87,7 @@ public abstract class GDBTrainer extends DatasetTrainer<ModelsComposition, Doubl
}
/** {@inheritDoc} */
- @Override public <K, V> ModelsComposition fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> ModelsComposition fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> preprocessor) {
return updateModel(null, datasetBuilder, preprocessor);
}
@@ -186,11 +186,13 @@ public abstract class GDBTrainer extends DatasetTrainer<ModelsComposition, Doubl
LearningEnvironmentBuilder envBuilder,
DatasetBuilder<K, V> builder,
Preprocessor<K, V> preprocessor) {
+ learningEnvironment().initDeployingContext(preprocessor);
try (Dataset<EmptyContext, DecisionTreeData> dataset = builder.build(
envBuilder,
new EmptyContextBuilder<>(),
- new DecisionTreeDataBuilder<>(preprocessor, false)
+ new DecisionTreeDataBuilder<>(preprocessor, false),
+ learningEnvironment()
)) {
IgniteBiTuple<Double, Long> meanTuple = dataset.compute(
data -> {
@@ -241,7 +243,7 @@ public abstract class GDBTrainer extends DatasetTrainer<ModelsComposition, Doubl
/**
* GDB model.
*/
- public static class GDBModel extends ModelsComposition {
+ public static final class GDBModel extends ModelsComposition {
/** Serial version uid. */
private static final long serialVersionUID = 3476661240155508004L;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceChecker.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceChecker.java
index 57c8047..26db23b 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceChecker.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceChecker.java
@@ -26,6 +26,7 @@ import org.apache.ignite.ml.dataset.primitive.FeatureMatrixWithLabelsOnHeapData;
import org.apache.ignite.ml.dataset.primitive.FeatureMatrixWithLabelsOnHeapDataBuilder;
import org.apache.ignite.ml.dataset.primitive.builder.context.EmptyContextBuilder;
import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.environment.LearningEnvironment;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -70,7 +71,6 @@ public abstract class ConvergenceChecker<K, V> implements Serializable {
IgniteFunction<Double, Double> externalLbToInternalMapping, Loss loss,
DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> preprocessor, double precision) {
-
assert precision < 1 && precision >= 0;
this.sampleSize = sampleSize;
@@ -91,10 +91,14 @@ public abstract class ConvergenceChecker<K, V> implements Serializable {
LearningEnvironmentBuilder envBuilder,
DatasetBuilder<K, V> datasetBuilder,
ModelsComposition currMdl) {
+ LearningEnvironment environment = envBuilder.buildForTrainer();
+ environment.initDeployingContext(preprocessor);
+
try (Dataset<EmptyContext, FeatureMatrixWithLabelsOnHeapData> dataset = datasetBuilder.build(
envBuilder,
new EmptyContextBuilder<>(),
- new FeatureMatrixWithLabelsOnHeapDataBuilder<>(preprocessor)
+ new FeatureMatrixWithLabelsOnHeapDataBuilder<>(preprocessor),
+ environment
)) {
return isConverged(dataset, currMdl);
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/parallel/ModelsParallelComposition.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/parallel/ModelsParallelComposition.java
index 601b639..fdf573f 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/parallel/ModelsParallelComposition.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/parallel/ModelsParallelComposition.java
@@ -30,7 +30,7 @@ import org.apache.ignite.ml.IgniteModel;
* @param <I> Type of submodel input.
* @param <O> Type of submodel output.
*/
-public class ModelsParallelComposition<I, O> implements IgniteModel<I, List<O>> {
+public final class ModelsParallelComposition<I, O> implements IgniteModel<I, List<O>> {
/** List of submodels. */
private final List<IgniteModel<I, O>> submodels;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/parallel/TrainersParallelComposition.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/parallel/TrainersParallelComposition.java
index ceaaf77..f88ae8f 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/parallel/TrainersParallelComposition.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/parallel/TrainersParallelComposition.java
@@ -78,7 +78,7 @@ public class TrainersParallelComposition<I, O, L> extends DatasetTrainer<IgniteM
}
/** {@inheritDoc} */
- @Override public <K, V> IgniteModel<I, List<O>> fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> IgniteModel<I, List<O>> fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> preprocessor) {
List<IgniteSupplier<IgniteModel<I, O>>> tasks = trainers.stream()
.map(tr -> (IgniteSupplier<IgniteModel<I, O>>)(() -> tr.fit(datasetBuilder, preprocessor)))
@@ -95,6 +95,8 @@ public class TrainersParallelComposition<I, O, L> extends DatasetTrainer<IgniteM
@Override public <K, V> IgniteModel<I, List<O>> update(IgniteModel<I, List<O>> mdl,
DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> preprocessor) {
+ learningEnvironment().initDeployingContext(preprocessor);
+
ModelsParallelComposition<I, O> typedMdl = (ModelsParallelComposition<I, O>)mdl;
assert typedMdl.submodels().size() == trainers.size();
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/sequential/ModelsSequentialComposition.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/sequential/ModelsSequentialComposition.java
index b82f25e..1fd8574 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/sequential/ModelsSequentialComposition.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/sequential/ModelsSequentialComposition.java
@@ -30,7 +30,7 @@ import org.apache.ignite.ml.math.functions.IgniteFunction;
* @param <O1> Type of output of the first model (and input of second).
* @param <O2> Type of output of the second model.
*/
-public class ModelsSequentialComposition<I, O1, O2> implements IgniteModel<I, O2> {
+public final class ModelsSequentialComposition<I, O1, O2> implements IgniteModel<I, O2> {
/** First model. */
private IgniteModel<I, O1> mdl1;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/sequential/TrainersSequentialComposition.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/sequential/TrainersSequentialComposition.java
index d4cbe85..b05ae2b 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/sequential/TrainersSequentialComposition.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/combinators/sequential/TrainersSequentialComposition.java
@@ -83,7 +83,7 @@ public class TrainersSequentialComposition<I, O1, O2, L> extends DatasetTrainer<
}
/** {@inheritDoc} */
- @Override public <K, V> ModelsSequentialComposition<I, O1, O2> fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> ModelsSequentialComposition<I, O1, O2> fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> preprocessor) {
IgniteModel<I, O1> mdl1 = tr1.fit(datasetBuilder, preprocessor);
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/MeanValuePredictionsAggregator.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/MeanValuePredictionsAggregator.java
index 01e693d..2f66a3a 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/MeanValuePredictionsAggregator.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/MeanValuePredictionsAggregator.java
@@ -23,7 +23,7 @@ import org.apache.ignite.internal.util.typedef.internal.A;
/**
* Predictions aggregator returning the mean value of predictions.
*/
-public class MeanValuePredictionsAggregator implements PredictionsAggregator {
+public final class MeanValuePredictionsAggregator implements PredictionsAggregator {
/** {@inheritDoc} */
@Override public Double apply(double[] estimations) {
A.notEmpty(estimations, "estimations vector");
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/OnMajorityPredictionsAggregator.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/OnMajorityPredictionsAggregator.java
index cd84a7e..995faad 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/OnMajorityPredictionsAggregator.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/OnMajorityPredictionsAggregator.java
@@ -25,7 +25,7 @@ import org.apache.ignite.internal.util.typedef.internal.A;
/**
* Predictions aggregator returning the most frequently prediction.
*/
-public class OnMajorityPredictionsAggregator implements PredictionsAggregator {
+public final class OnMajorityPredictionsAggregator implements PredictionsAggregator {
/** {@inheritDoc} */
@Override public Double apply(double[] estimations) {
A.notEmpty(estimations, "estimations vector");
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/WeightedPredictionsAggregator.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/WeightedPredictionsAggregator.java
index 15b5d7f..2642bf8 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/WeightedPredictionsAggregator.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/WeightedPredictionsAggregator.java
@@ -23,7 +23,7 @@ import org.apache.ignite.internal.util.typedef.internal.A;
* Predictions aggregator returning weighted plus of predictions.
* result(p1, ..., pn) = bias + p1*w1 + ... + pn*wn
*/
-public class WeightedPredictionsAggregator implements PredictionsAggregator {
+public final class WeightedPredictionsAggregator implements PredictionsAggregator {
/** Weights for predictions. */
private final double[] weights;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/stacking/StackedDatasetTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/stacking/StackedDatasetTrainer.java
index a74944c..b8324fd 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/stacking/StackedDatasetTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/stacking/StackedDatasetTrainer.java
@@ -230,7 +230,7 @@ public class StackedDatasetTrainer<IS, IA, O, AM extends IgniteModel<IA, O>, L>
}
/** {@inheritDoc} */
- @Override public <K, V> StackedModel<IS, IA, O, AM> fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> StackedModel<IS, IA, O, AM> fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> preprocessor) {
return new StackedModel<>(getTrainer().fit(datasetBuilder, preprocessor));
@@ -239,7 +239,7 @@ public class StackedDatasetTrainer<IS, IA, O, AM extends IgniteModel<IA, O>, L>
/** {@inheritDoc} */
@Override public <K, V> StackedModel<IS, IA, O, AM> update(StackedModel<IS, IA, O, AM> mdl,
DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> preprocessor) {
-
+ learningEnvironment().initDeployingContext(preprocessor);
return new StackedModel<>(getTrainer().update(mdl, datasetBuilder, preprocessor));
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/stacking/StackedModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/stacking/StackedModel.java
index 5793c2e..bd5c60c 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/stacking/StackedModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/stacking/StackedModel.java
@@ -38,7 +38,7 @@ import org.apache.ignite.ml.composition.combinators.parallel.ModelsParallelCompo
* @param <O> Type of aggregator model output.
* @param <AM> Type of aggregator model.
*/
-public class StackedModel<IS, IA, O, AM extends IgniteModel<IA, O>> implements IgniteModel<IS, O> {
+public final class StackedModel<IS, IA, O, AM extends IgniteModel<IA, O>> implements IgniteModel<IS, O> {
/** Model to wrap. */
private IgniteModel<IS, O> mdl;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/DatasetBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/DatasetBuilder.java
index c826a40..32054af 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/DatasetBuilder.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/DatasetBuilder.java
@@ -21,6 +21,7 @@ import java.io.Serializable;
import org.apache.ignite.lang.IgniteBiPredicate;
import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.environment.LearningEnvironment;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
import org.apache.ignite.ml.trainers.transformers.BaggingUpstreamTransformer;
@@ -46,12 +47,14 @@ public interface DatasetBuilder<K, V> {
* @param partDataBuilder Partition {@code data} builder.
* @param <C> Type of a partition {@code context}.
* @param <D> Type of a partition {@code data}.
+ * @param localLearningEnv Local learning environment.
* @return Dataset.
*/
public <C extends Serializable, D extends AutoCloseable> Dataset<C, D> build(
LearningEnvironmentBuilder envBuilder,
PartitionContextBuilder<K, V, C> partCtxBuilder,
- PartitionDataBuilder<K, V, C, D> partDataBuilder);
+ PartitionDataBuilder<K, V, C, D> partDataBuilder,
+ LearningEnvironment localLearningEnv);
/**
* Returns new instance of {@link DatasetBuilder} with new {@link UpstreamTransformerBuilder} added
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/DatasetFactory.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/DatasetFactory.java
index a0d550a..c6d7343 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/DatasetFactory.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/DatasetFactory.java
@@ -31,6 +31,7 @@ import org.apache.ignite.ml.dataset.primitive.builder.data.SimpleLabeledDatasetD
import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
import org.apache.ignite.ml.dataset.primitive.data.SimpleDatasetData;
import org.apache.ignite.ml.dataset.primitive.data.SimpleLabeledDatasetData;
+import org.apache.ignite.ml.environment.LearningEnvironment;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
import org.apache.ignite.ml.preprocessing.Preprocessor;
@@ -74,6 +75,7 @@ public class DatasetFactory {
* @param datasetBuilder Dataset builder.
* @param partCtxBuilder Partition {@code context} builder.
* @param partDataBuilder Partition {@code data} builder.
+ * @param environment Local learning environment.
* @param <K> Type of a key in {@code upstream} data.
* @param <V> ype of a value in {@code upstream} data.
* @param <C> Type of a partition {@code context}.
@@ -84,11 +86,13 @@ public class DatasetFactory {
DatasetBuilder<K, V> datasetBuilder,
LearningEnvironmentBuilder envBuilder,
PartitionContextBuilder<K, V, C> partCtxBuilder,
- PartitionDataBuilder<K, V, C, D> partDataBuilder) {
+ PartitionDataBuilder<K, V, C, D> partDataBuilder,
+ LearningEnvironment environment) {
return datasetBuilder.build(
envBuilder,
partCtxBuilder,
- partDataBuilder
+ partDataBuilder,
+ environment
);
}
@@ -110,10 +114,15 @@ public class DatasetFactory {
DatasetBuilder<K, V> datasetBuilder,
PartitionContextBuilder<K, V, C> partCtxBuilder,
PartitionDataBuilder<K, V, C, D> partDataBuilder) {
+
+ LearningEnvironment environment = LearningEnvironmentBuilder.defaultBuilder().buildForTrainer();
+ environment.deployingContext().initByClientObject(partDataBuilder);
+
return datasetBuilder.build(
LearningEnvironmentBuilder.defaultBuilder(),
partCtxBuilder,
- partDataBuilder
+ partDataBuilder,
+ environment
);
}
@@ -127,6 +136,7 @@ public class DatasetFactory {
* @param envBuilder Learning environment builder.
* @param partCtxBuilder Partition {@code context} builder.
* @param partDataBuilder Partition {@code data} builder.
+ * @param environment Local learning environment.
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
* @param <C> Type of a partition {@code context}.
@@ -137,12 +147,14 @@ public class DatasetFactory {
Ignite ignite, IgniteCache<K, V> upstreamCache,
LearningEnvironmentBuilder envBuilder,
PartitionContextBuilder<K, V, C> partCtxBuilder,
- PartitionDataBuilder<K, V, C, D> partDataBuilder) {
+ PartitionDataBuilder<K, V, C, D> partDataBuilder,
+ LearningEnvironment environment) {
return create(
new CacheBasedDatasetBuilder<>(ignite, upstreamCache),
envBuilder,
partCtxBuilder,
- partDataBuilder
+ partDataBuilder,
+ environment
);
}
@@ -191,11 +203,16 @@ public class DatasetFactory {
LearningEnvironmentBuilder envBuilder,
PartitionContextBuilder<K, V, C> partCtxBuilder,
Preprocessor<K, V> featureExtractor) {
+
+ LearningEnvironment environment = LearningEnvironmentBuilder.defaultBuilder().buildForTrainer();
+ environment.initDeployingContext(featureExtractor);
+
return create(
datasetBuilder,
envBuilder,
partCtxBuilder,
- new SimpleDatasetDataBuilder<>(featureExtractor)
+ new SimpleDatasetDataBuilder<>(featureExtractor),
+ environment
).wrap(SimpleDataset::new);
}
@@ -248,11 +265,16 @@ public class DatasetFactory {
LearningEnvironmentBuilder envBuilder,
PartitionContextBuilder<K, V, C> partCtxBuilder,
Preprocessor<K, V> vectorizer) {
+
+ LearningEnvironment environment = LearningEnvironmentBuilder.defaultBuilder().buildForTrainer();
+ environment.initDeployingContext(vectorizer);
+
return create(
datasetBuilder,
envBuilder,
partCtxBuilder,
- new SimpleLabeledDatasetDataBuilder<>(vectorizer)
+ new SimpleLabeledDatasetDataBuilder<>(vectorizer),
+ environment
).wrap(SimpleLabeledDataset::new);
}
@@ -419,6 +441,7 @@ public class DatasetFactory {
* @param partCtxBuilder Partition {@code context} builder.
* @param envBuilder Learning environment builder.
* @param partDataBuilder Partition {@code data} builder.
+ * @param environment Local learning environment.
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
* @param <C> Type of a partition {@code context}.
@@ -429,12 +452,14 @@ public class DatasetFactory {
Map<K, V> upstreamMap,
LearningEnvironmentBuilder envBuilder,
int partitions, PartitionContextBuilder<K, V, C> partCtxBuilder,
- PartitionDataBuilder<K, V, C, D> partDataBuilder) {
+ PartitionDataBuilder<K, V, C, D> partDataBuilder,
+ LearningEnvironment environment) {
return create(
new LocalDatasetBuilder<>(upstreamMap, partitions),
envBuilder,
partCtxBuilder,
- partDataBuilder
+ partDataBuilder,
+ environment
);
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/Vectorizer.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/Vectorizer.java
index 67a78fb..9030b5c 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/Vectorizer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/Vectorizer.java
@@ -238,7 +238,7 @@ public abstract class Vectorizer<K, V, C extends Serializable, L> implements Fea
* @param <C> Type of coordinate.
* @param <L> Type od label.
*/
- public static class VectorizerAdapter<K, V, C extends Serializable, L> extends Vectorizer<K, V, C, L> {
+ public abstract static class VectorizerAdapter<K, V, C extends Serializable, L> extends Vectorizer<K, V, C, L> {
/** {@inheritDoc} */
@Override protected Double feature(C coord, K key, V value) {
throw new IllegalStateException();
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/BinaryObjectVectorizer.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/BinaryObjectVectorizer.java
index d4502bd..f4a49c6 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/BinaryObjectVectorizer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/BinaryObjectVectorizer.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ml.dataset.feature.extractor.impl;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.stream.Collectors;
@@ -24,6 +25,7 @@ import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.internal.binary.BinaryUtils;
import org.apache.ignite.internal.binary.GridBinaryMarshaller;
import org.apache.ignite.ml.dataset.feature.extractor.ExtractionUtils;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.impl.SparseVector;
@@ -32,7 +34,8 @@ import org.apache.ignite.ml.math.primitives.vector.impl.SparseVector;
*
* @param <K> Type of key.
*/
-public class BinaryObjectVectorizer<K> extends ExtractionUtils.StringCoordVectorizer<K, BinaryObject> {
+public final class BinaryObjectVectorizer<K> extends ExtractionUtils.StringCoordVectorizer<K, BinaryObject>
+ implements DeployableObject {
/** Serial version uid. */
private static final long serialVersionUID = 2152161240934492838L;
@@ -139,4 +142,9 @@ public class BinaryObjectVectorizer<K> extends ExtractionUtils.StringCoordVector
return value;
}
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.emptyList();
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/DoubleArrayVectorizer.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/DoubleArrayVectorizer.java
index fb36b59..ad9a947 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/DoubleArrayVectorizer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/DoubleArrayVectorizer.java
@@ -17,14 +17,18 @@
package org.apache.ignite.ml.dataset.feature.extractor.impl;
+import java.util.Collections;
+import java.util.List;
import org.apache.ignite.ml.dataset.feature.extractor.ExtractionUtils;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
/**
* Vectorizer on arrays of doubles.
*
* @param <K> Key type.
*/
-public class DoubleArrayVectorizer<K> extends ExtractionUtils.ArrayLikeVectorizer<K, double[]> {
+public final class DoubleArrayVectorizer<K> extends ExtractionUtils.ArrayLikeVectorizer<K, double[]>
+ implements DeployableObject {
/** Serial version uid. */
private static final long serialVersionUID = -1177109334215177722L;
@@ -46,4 +50,9 @@ public class DoubleArrayVectorizer<K> extends ExtractionUtils.ArrayLikeVectorize
@Override protected int sizeOf(K key, double[] value) {
return value.length;
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.emptyList();
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/DummyVectorizer.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/DummyVectorizer.java
index 8cde8d1..b5af02a 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/DummyVectorizer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/DummyVectorizer.java
@@ -18,7 +18,10 @@
package org.apache.ignite.ml.dataset.feature.extractor.impl;
import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
import org.apache.ignite.ml.dataset.feature.extractor.ExtractionUtils;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.primitives.vector.Vector;
/**
@@ -26,7 +29,7 @@ import org.apache.ignite.ml.math.primitives.vector.Vector;
*
* @param <K> Type of key.
*/
-public class DummyVectorizer<K> extends ExtractionUtils.ArrayLikeVectorizer<K, Vector> {
+public final class DummyVectorizer<K> extends ExtractionUtils.ArrayLikeVectorizer<K, Vector> implements DeployableObject {
/** Serial version uid. */
private static final long serialVersionUID = -6225354615212148224L;
@@ -49,4 +52,8 @@ public class DummyVectorizer<K> extends ExtractionUtils.ArrayLikeVectorizer<K, V
return value.size();
}
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.emptyList();
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/LabeledDummyVectorizer.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/LabeledDummyVectorizer.java
index 2c1e704..b070e28 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/LabeledDummyVectorizer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/feature/extractor/impl/LabeledDummyVectorizer.java
@@ -17,19 +17,20 @@
package org.apache.ignite.ml.dataset.feature.extractor.impl;
-import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
-import org.apache.ignite.ml.structures.LabeledVector;
-
+import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
+import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
+import org.apache.ignite.ml.structures.LabeledVector;
/**
* Vectorizer on LabeledVector.
*
* @param <K> Type of key.
*/
-public class LabeledDummyVectorizer<K, L> extends Vectorizer<K, LabeledVector<L>, Integer, L> {
+public class LabeledDummyVectorizer<K, L> extends Vectorizer<K, LabeledVector<L>, Integer, L> implements DeployableObject {
/** Serial version uid. */
private static final long serialVersionUID = -6225354615212148224L;
@@ -62,4 +63,9 @@ public class LabeledDummyVectorizer<K, L> extends Vectorizer<K, LabeledVector<L>
@Override protected List<Integer> allCoords(K key, LabeledVector<L> value) {
return IntStream.range(0, value.features().size()).boxed().collect(Collectors.toList());
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.emptyList();
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDataset.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDataset.java
index d0253fe..43852e6 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDataset.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDataset.java
@@ -48,9 +48,6 @@ import org.apache.ignite.ml.math.functions.IgniteTriFunction;
*/
public class CacheBasedDataset<K, V, C extends Serializable, D extends AutoCloseable>
implements Dataset<C, D> {
- /** Number of retries for the case when one of partitions not found on the node where computation is performed. */
- private static final int RETRIES = 15 * 60;
-
/** Retry interval (ms) for the case when one of partitions not found on the node where computation is performed. */
private static final int RETRY_INTERVAL = 1000;
@@ -81,6 +78,14 @@ public class CacheBasedDataset<K, V, C extends Serializable, D extends AutoClose
/** Upstream keep binary. */
private final boolean upstreamKeepBinary;
+ /** Number of retries for the case when one of partitions not found on the node where computation is performed. */
+ private final int retries;
+
+ /**
+ * Client-side learning environment.
+ */
+ private final LearningEnvironment localLearningEnv;
+
/**
* Constructs a new instance of dataset based on Ignite Cache, which is used as {@code upstream} and as reliable storage for
* partition {@code context} as well.
@@ -92,6 +97,8 @@ public class CacheBasedDataset<K, V, C extends Serializable, D extends AutoClose
* @param datasetCache Ignite Cache with partition {@code context}.
* @param partDataBuilder Partition {@code data} builder.
* @param datasetId Dataset ID.
+ * @param localLearningEnv Local learning environment.
+ * @param retriesCnt Number of retries for the case when one of partitions not found on the node where computation is performed.
*/
public CacheBasedDataset(
Ignite ignite,
@@ -102,7 +109,10 @@ public class CacheBasedDataset<K, V, C extends Serializable, D extends AutoClose
LearningEnvironmentBuilder envBuilder,
PartitionDataBuilder<K, V, C, D> partDataBuilder,
UUID datasetId,
- boolean upstreamKeepBinary) {
+ boolean upstreamKeepBinary,
+ LearningEnvironment localLearningEnv,
+ int retriesCnt) {
+
this.ignite = ignite;
this.upstreamCache = upstreamCache;
this.filter = filter;
@@ -112,6 +122,8 @@ public class CacheBasedDataset<K, V, C extends Serializable, D extends AutoClose
this.envBuilder = envBuilder;
this.datasetId = datasetId;
this.upstreamKeepBinary = upstreamKeepBinary;
+ this.localLearningEnv = localLearningEnv;
+ this.retries = retriesCnt;
}
/** {@inheritDoc} */
@@ -192,7 +204,7 @@ public class CacheBasedDataset<K, V, C extends Serializable, D extends AutoClose
*/
private <R> R computeForAllPartitions(IgniteFunction<Integer, R> fun, IgniteBinaryOperator<R> reduce, R identity) {
Collection<String> cacheNames = Arrays.asList(datasetCache.getName(), upstreamCache.getName());
- Collection<R> results = ComputeUtils.affinityCallWithRetries(ignite, cacheNames, fun, RETRIES, RETRY_INTERVAL);
+ Collection<R> results = ComputeUtils.affinityCallWithRetries(ignite, cacheNames, fun, retries, RETRY_INTERVAL, localLearningEnv.deployingContext());
R res = identity;
for (R partRes : results)
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetBuilder.java
index ea11943..0394ebd 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetBuilder.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetBuilder.java
@@ -30,6 +30,7 @@ import org.apache.ignite.ml.dataset.PartitionDataBuilder;
import org.apache.ignite.ml.dataset.UpstreamTransformerBuilder;
import org.apache.ignite.ml.dataset.impl.cache.util.ComputeUtils;
import org.apache.ignite.ml.dataset.impl.cache.util.DatasetAffinityFunctionWrapper;
+import org.apache.ignite.ml.environment.LearningEnvironment;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
/**
@@ -40,8 +41,8 @@ import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
* @param <V> Type of a value in {@code upstream} data.
*/
public class CacheBasedDatasetBuilder<K, V> implements DatasetBuilder<K, V> {
- /** Number of retries for the case when one of partitions not found on the node where loading is performed. */
- private static final int RETRIES = 15 * 60;
+ /** Default number of retries for the case when one of partitions not found on the node where loading is performed. */
+ public static final int DEFAULT_NUMBER_OF_RETRIES = 15 * 60;
/** Retry interval (ms) for the case when one of partitions not found on the node where loading is performed. */
private static final int RETRY_INTERVAL = 1000;
@@ -64,6 +65,9 @@ public class CacheBasedDatasetBuilder<K, V> implements DatasetBuilder<K, V> {
/** Upstream keep binary. */
private final boolean upstreamKeepBinary;
+ /** Number of retries for the case when one of partitions not found on the node where loading is performed. */
+ private final int retries;
+
/**
* Constructs a new instance of cache based dataset builder that makes {@link CacheBasedDataset} with default
* predicate that passes all upstream entries to dataset.
@@ -97,7 +101,7 @@ public class CacheBasedDatasetBuilder<K, V> implements DatasetBuilder<K, V> {
IgniteCache<K, V> upstreamCache,
IgniteBiPredicate<K, V> filter,
UpstreamTransformerBuilder transformerBuilder) {
- this(ignite, upstreamCache, filter, transformerBuilder, false);
+ this(ignite, upstreamCache, filter, transformerBuilder, false, DEFAULT_NUMBER_OF_RETRIES);
}
/**
@@ -108,17 +112,20 @@ public class CacheBasedDatasetBuilder<K, V> implements DatasetBuilder<K, V> {
* @param filter Filter.
* @param transformerBuilder Transformer builder.
* @param isKeepBinary Is keep binary for upstream cache.
+ * @param retries Number of retries for the case when one of partitions not found on the node where loading is performed.
*/
public CacheBasedDatasetBuilder(Ignite ignite,
IgniteCache<K, V> upstreamCache,
IgniteBiPredicate<K, V> filter,
UpstreamTransformerBuilder transformerBuilder,
- Boolean isKeepBinary){
+ Boolean isKeepBinary,
+ int retries){
this.ignite = ignite;
this.upstreamCache = upstreamCache;
this.filter = filter;
this.transformerBuilder = transformerBuilder;
this.upstreamKeepBinary = isKeepBinary;
+ this.retries = retries;
}
/** {@inheritDoc} */
@@ -126,7 +133,9 @@ public class CacheBasedDatasetBuilder<K, V> implements DatasetBuilder<K, V> {
@Override public <C extends Serializable, D extends AutoCloseable> CacheBasedDataset<K, V, C, D> build(
LearningEnvironmentBuilder envBuilder,
PartitionContextBuilder<K, V, C> partCtxBuilder,
- PartitionDataBuilder<K, V, C, D> partDataBuilder) {
+ PartitionDataBuilder<K, V, C, D> partDataBuilder,
+ LearningEnvironment localLearningEnv) {
+
UUID datasetId = UUID.randomUUID();
// Retrieves affinity function of the upstream Ignite Cache.
@@ -149,12 +158,13 @@ public class CacheBasedDatasetBuilder<K, V> implements DatasetBuilder<K, V> {
datasetCache.getName(),
partCtxBuilder,
envBuilder,
- RETRIES,
+ retries,
RETRY_INTERVAL,
- upstreamKeepBinary
+ upstreamKeepBinary,
+ localLearningEnv.deployingContext()
);
- return new CacheBasedDataset<>(ignite, upstreamCache, filter, transformerBuilder, datasetCache, envBuilder, partDataBuilder, datasetId, upstreamKeepBinary);
+ return new CacheBasedDataset<>(ignite, upstreamCache, filter, transformerBuilder, datasetCache, envBuilder, partDataBuilder, datasetId, upstreamKeepBinary, localLearningEnv, retries);
}
/** {@inheritDoc} */
@@ -174,6 +184,16 @@ public class CacheBasedDatasetBuilder<K, V> implements DatasetBuilder<K, V> {
* @param isKeepBinary Is keep binary.
*/
public CacheBasedDatasetBuilder<K, V> withKeepBinary(boolean isKeepBinary){
- return new CacheBasedDatasetBuilder<K, V>(ignite, upstreamCache, filter, transformerBuilder, isKeepBinary);
+ return new CacheBasedDatasetBuilder<K, V>(ignite, upstreamCache, filter, transformerBuilder, isKeepBinary, retries);
+ }
+
+ /**
+ * Sets number of retries. 15 * 60 by default.
+ *
+ * @param retries Number of retries.
+ * @return CacheBasedDatasetBuilder instance.
+ */
+ public CacheBasedDatasetBuilder<K, V> withRetriesNumber(int retries) {
+ return new CacheBasedDatasetBuilder<K, V>(ignite, upstreamCache, filter, transformerBuilder, upstreamKeepBinary, retries);
}
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/util/ComputeUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/util/ComputeUtils.java
index 868245d..0ce1511 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/util/ComputeUtils.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/util/ComputeUtils.java
@@ -38,7 +38,9 @@ import org.apache.ignite.cache.affinity.Affinity;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.internal.util.lang.GridPeerDeployAware;
import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.lang.IgniteCallable;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.ml.dataset.PartitionContextBuilder;
import org.apache.ignite.ml.dataset.PartitionDataBuilder;
@@ -47,6 +49,7 @@ import org.apache.ignite.ml.dataset.UpstreamTransformer;
import org.apache.ignite.ml.dataset.UpstreamTransformerBuilder;
import org.apache.ignite.ml.environment.LearningEnvironment;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
+import org.apache.ignite.ml.environment.deploy.DeployingContext;
import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.ml.util.Utils;
@@ -71,11 +74,12 @@ public class ComputeUtils {
* @param fun Function to be applied on all partitions.
* @param retries Number of retries for the case when one of partitions not found on the node.
* @param interval Interval of retries for the case when one of partitions not found on the node.
+ * @param deployingContext Deploy context of user-defined classes for peer class loading.
* @param <R> Type of a result.
* @return Collection of results.
*/
public static <R> Collection<R> affinityCallWithRetries(Ignite ignite, Collection<String> cacheNames,
- IgniteFunction<Integer, R> fun, int retries, int interval) {
+ IgniteFunction<Integer, R> fun, int retries, int interval, DeployingContext deployingContext) {
assert !cacheNames.isEmpty();
assert interval >= 0;
@@ -98,7 +102,10 @@ public class ComputeUtils {
futures.put(
currPart,
- ignite.compute(clusterGrp).affinityCallAsync(cacheNames, currPart, () -> fun.apply(currPart))
+ ignite.compute(clusterGrp).affinityCallAsync(
+ cacheNames, currPart,
+ new DeployableCallable<>(deployingContext, part, fun)
+ )
);
}
@@ -131,12 +138,13 @@ public class ComputeUtils {
* @param cacheNames Collection of cache names.
* @param fun Function to be applied on all partitions.
* @param retries Number of retries for the case when one of partitions not found on the node.
+ * @param deployingContext Deploy context.
* @param <R> Type of a result.
* @return Collection of results.
*/
public static <R> Collection<R> affinityCallWithRetries(Ignite ignite, Collection<String> cacheNames,
- IgniteFunction<Integer, R> fun, int retries) {
- return affinityCallWithRetries(ignite, cacheNames, fun, retries, 0);
+ IgniteFunction<Integer, R> fun, int retries, DeployingContext deployingContext) {
+ return affinityCallWithRetries(ignite, cacheNames, fun, retries, 0, deployingContext);
}
/**
@@ -265,6 +273,7 @@ public class ComputeUtils {
* @param ctxBuilder Partition {@code context} builder.
* @param envBuilder Environment builder.
* @param isKeepBinary Support of binary objects.
+ * @param deployingContext Deploy context.
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
* @param <C> Type of a partition {@code context}.
@@ -279,7 +288,9 @@ public class ComputeUtils {
LearningEnvironmentBuilder envBuilder,
int retries,
int interval,
- boolean isKeepBinary) {
+ boolean isKeepBinary,
+ DeployingContext deployingContext) {
+
affinityCallWithRetries(ignite, Arrays.asList(datasetCacheName, upstreamCacheName), part -> {
Ignite locIgnite = Ignition.localIgnite();
LearningEnvironment env = envBuilder.buildForWorker(part);
@@ -320,7 +331,7 @@ public class ComputeUtils {
datasetCache.put(part, ctx);
return part;
- }, retries, interval);
+ }, retries, interval, deployingContext);
}
/**
@@ -388,4 +399,47 @@ public class ComputeUtils {
return res;
}
+
+ /**
+ * Callable that contains deploy context and can pass missing classes
+ * during learning session by p2p deployment.
+ * @param <C> Type of callable result.
+ */
+ private static class DeployableCallable<C> implements GridPeerDeployAware, IgniteCallable<C> {
+ /** Fun. */
+ private final IgniteFunction<Integer, C> fun;
+
+ /** Partition. */
+ private final int part;
+
+ /** Deploy context. */
+ private transient DeployingContext deployingContext;
+
+ /**
+ * Creates an instance of DeployableCallable.
+ * @param deployingContext Deploy context.
+ * @param part Partition.
+ * @param fun Callable function.
+ */
+ public DeployableCallable(DeployingContext deployingContext, int part, IgniteFunction<Integer, C> fun) {
+ this.fun = fun;
+ this.deployingContext = deployingContext;
+ this.part = part;
+ }
+
+ /** {@inheritDoc} */
+ @Override public C call() throws Exception {
+ return fun.apply(part);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Class<?> deployClass() {
+ return deployingContext.userClass();
+ }
+
+ /** {@inheritDoc} */
+ @Override public ClassLoader classLoader() {
+ return deployingContext.clientClassLoader();
+ }
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/local/LocalDatasetBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/local/LocalDatasetBuilder.java
index 84f3e08..b00d480 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/local/LocalDatasetBuilder.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/local/LocalDatasetBuilder.java
@@ -101,7 +101,9 @@ public class LocalDatasetBuilder<K, V> implements DatasetBuilder<K, V> {
/** {@inheritDoc} */
@Override public <C extends Serializable, D extends AutoCloseable> LocalDataset<C, D> build(
LearningEnvironmentBuilder envBuilder,
- PartitionContextBuilder<K, V, C> partCtxBuilder, PartitionDataBuilder<K, V, C, D> partDataBuilder) {
+ PartitionContextBuilder<K, V, C> partCtxBuilder, PartitionDataBuilder<K, V, C, D> partDataBuilder,
+ LearningEnvironment learningEnvironment) {
+
List<C> ctxList = new ArrayList<>();
List<D> dataList = new ArrayList<>();
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/environment/DefaultLearningEnvironmentBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/environment/DefaultLearningEnvironmentBuilder.java
index ab38e6e..2b0e13d 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/environment/DefaultLearningEnvironmentBuilder.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/environment/DefaultLearningEnvironmentBuilder.java
@@ -18,6 +18,7 @@
package org.apache.ignite.ml.environment;
import java.util.Random;
+import org.apache.ignite.ml.environment.deploy.DeployingContext;
import org.apache.ignite.ml.environment.logging.MLLogger;
import org.apache.ignite.ml.environment.logging.NoOpLogger;
import org.apache.ignite.ml.environment.parallelism.DefaultParallelismStrategy;
@@ -148,6 +149,9 @@ public class DefaultLearningEnvironmentBuilder implements LearningEnvironmentBui
/** Random numbers generator. */
private final Random randomNumGen;
+ /** Deploy context. */
+ private final DeployingContext deployingContext = DeployingContext.unitialized();
+
/**
* Creates an instance of LearningEnvironmentImpl.
*
@@ -199,5 +203,10 @@ public class DefaultLearningEnvironmentBuilder implements LearningEnvironmentBui
@Override public long dataTtl() {
return dataTtl;
}
+
+ /** {@inheritDoc} */
+ @Override public DeployingContext deployingContext() {
+ return deployingContext;
+ }
}
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/environment/LearningEnvironment.java b/modules/ml/src/main/java/org/apache/ignite/ml/environment/LearningEnvironment.java
index 8e4dce2..5cbe4ac 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/environment/LearningEnvironment.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/environment/LearningEnvironment.java
@@ -19,6 +19,7 @@ package org.apache.ignite.ml.environment;
import java.util.Random;
import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.environment.deploy.DeployingContext;
import org.apache.ignite.ml.environment.logging.MLLogger;
import org.apache.ignite.ml.environment.parallelism.ParallelismStrategy;
@@ -67,4 +68,21 @@ public interface LearningEnvironment {
* @return Partition data time-to-live in seconds (-1 for an infinite lifetime).
*/
public long dataTtl();
+
+ /**
+ * Returns deploy context instance.
+ *
+ * @return Deploy context.
+ */
+ public DeployingContext deployingContext();
+
+ /**
+ * Initializes deploying context by object representing current client computation
+ * with classes unknown for server side.
+ *
+ * @param clientSideObj Client side object.
+ */
+ public default void initDeployingContext(Object clientSideObj) {
+ deployingContext().initByClientObject(clientSideObj);
+ }
}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/environment/EnvironmentTestSuite.java b/modules/ml/src/main/java/org/apache/ignite/ml/environment/deploy/DeployableObject.java
similarity index 61%
copy from modules/ml/src/test/java/org/apache/ignite/ml/environment/EnvironmentTestSuite.java
copy to modules/ml/src/main/java/org/apache/ignite/ml/environment/deploy/DeployableObject.java
index ae9f490..08a3cbe 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/environment/EnvironmentTestSuite.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/environment/deploy/DeployableObject.java
@@ -15,19 +15,19 @@
* limitations under the License.
*/
-package org.apache.ignite.ml.environment;
+package org.apache.ignite.ml.environment.deploy;
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
+import java.util.List;
/**
- * Test suite for all tests located in org.apache.ignite.ml.trees package.
+ * Represents an final objects from Ignite ML library like models or preprocessors having
+ * dependencies that can be custom objects from client side.
+ *
+ * NOTE: this interface should be inherited only by Ignite ML classes.
*/
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
- LearningEnvironmentBuilderTest.class,
- LearningEnvironmentTest.class,
- PromiseTest.class
-})
-public class EnvironmentTestSuite {
+public interface DeployableObject {
+ /**
+ * Returns dependencies of this object that can be object with class defined by client side and unknown for server.
+ */
+ public List<Object> getDependencies();
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/environment/deploy/DeployingContext.java b/modules/ml/src/main/java/org/apache/ignite/ml/environment/deploy/DeployingContext.java
new file mode 100644
index 0000000..585410b
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/environment/deploy/DeployingContext.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.environment.deploy;
+
+/**
+ * Class represents user's class loading environment for specific remote job.
+ * This contexst is used to determine class loader for missing classes on worker node.
+ */
+public interface DeployingContext {
+ /**
+ * @return Class represents client-defined class.
+ */
+ public Class<?> userClass();
+
+ /**
+ * @return Client class loader.
+ */
+ public ClassLoader clientClassLoader();
+
+ /**
+ * Inits context by other context.
+ *
+ * @param other Other context.
+ */
+ public void init(DeployingContext other);
+
+ /**
+ * Inits context by any client-defined object.
+ *
+ * @param jobObject Specific class for job.
+ */
+ public void initByClientObject(Object jobObject);
+
+ /**
+ * @return Unitialized deploy context.
+ */
+ public static DeployingContext unitialized() {
+ return new DeployingContextImpl();
+ }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/environment/deploy/DeployingContextImpl.java b/modules/ml/src/main/java/org/apache/ignite/ml/environment/deploy/DeployingContextImpl.java
new file mode 100644
index 0000000..347ccdc
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/environment/deploy/DeployingContextImpl.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.environment.deploy;
+
+import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Default implementation of {@link DeployingContext} class.
+ */
+public class DeployingContextImpl implements DeployingContext {
+ /** Logger. */
+ private static final Logger logger = LoggerFactory.getLogger(DeployingContextImpl.class);
+
+ /** Preprocessor class. */
+ private transient Class<?> preprocessorClass;
+
+ /** Client class loader. */
+ private transient ClassLoader clientClassLoader;
+
+ /** {@inheritDoc} */
+ @Override public Class<?> userClass() {
+ return preprocessorClass == null ? this.getClass() : preprocessorClass;
+ }
+
+ /** {@inheritDoc} */
+ @Override public ClassLoader clientClassLoader() {
+ return clientClassLoader == null ? this.getClass().getClassLoader() : clientClassLoader;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initByClientObject(Object jobObj) {
+ if (jobObj == null) {
+ logger.warn("Attempt to initialize deploy context by null");
+ return;
+ }
+
+ if (preprocessorClass != null)
+ logger.warn("Reinitialize deploying context [class=" + jobObj.getClass().getName() + "]");
+
+ Object objectToDeploy = jobObj;
+ while (objectToDeploy instanceof DeployableObject) {
+ List<Object> deps = ((DeployableObject)objectToDeploy).getDependencies();
+ if (deps.isEmpty())
+ break;
+ else
+ objectToDeploy = deps.get(0);
+ }
+
+ assert objectToDeploy != null;
+ preprocessorClass = objectToDeploy.getClass();
+ clientClassLoader = preprocessorClass.getClassLoader();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void init(DeployingContext other) {
+ this.clientClassLoader = other.clientClassLoader();
+ this.preprocessorClass = other.userClass();
+ }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/environment/EnvironmentTestSuite.java b/modules/ml/src/main/java/org/apache/ignite/ml/environment/deploy/package-info.java
similarity index 68%
copy from modules/ml/src/test/java/org/apache/ignite/ml/environment/EnvironmentTestSuite.java
copy to modules/ml/src/main/java/org/apache/ignite/ml/environment/deploy/package-info.java
index ae9f490..1533d98 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/environment/EnvironmentTestSuite.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/environment/deploy/package-info.java
@@ -15,19 +15,7 @@
* limitations under the License.
*/
-package org.apache.ignite.ml.environment;
-
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-
/**
- * Test suite for all tests located in org.apache.ignite.ml.trees package.
+ * Package contains user-defined classes deploy support tools.
*/
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
- LearningEnvironmentBuilderTest.class,
- LearningEnvironmentTest.class,
- PromiseTest.class
-})
-public class EnvironmentTestSuite {
-}
+package org.apache.ignite.ml.environment.deploy;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNTrainer.java
index b4401c5..b5b2fe1 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNTrainer.java
@@ -50,7 +50,8 @@ public abstract class KNNTrainer<M extends KNNModel<Double>, Self extends KNNTra
protected boolean weighted;
/** {@inheritDoc} */
- @Override public <K, V> M fit(DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> preprocessor) {
+ @Override protected <K, V> M fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
+ Preprocessor<K, V> preprocessor) {
return updateModel(null, datasetBuilder, preprocessor);
}
@@ -65,7 +66,8 @@ public abstract class KNNTrainer<M extends KNNModel<Double>, Self extends KNNTra
Dataset<EmptyContext, SpatialIndex<Double>> knnDataset = datasetBuilder.build(
envBuilder,
new EmptyContextBuilder<>(),
- new KNNPartitionDataBuilder<>(preprocessor, idxType, distanceMeasure)
+ new KNNPartitionDataBuilder<>(preprocessor, idxType, distanceMeasure),
+ learningEnvironment()
);
return convertDatasetIntoModel(knnDataset);
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/NNClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/NNClassificationModel.java
index 3059ce4..a48859a 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/NNClassificationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/NNClassificationModel.java
@@ -28,6 +28,7 @@ import org.apache.ignite.ml.Exportable;
import org.apache.ignite.ml.Exporter;
import org.apache.ignite.ml.IgniteModel;
import org.apache.ignite.ml.knn.ann.KNNModelFormat;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.distances.DistanceMeasure;
import org.apache.ignite.ml.math.distances.EuclideanDistance;
import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -40,7 +41,8 @@ import org.jetbrains.annotations.NotNull;
* Common methods and fields for all kNN and aNN models
* to predict label based on neighbours' labels.
*/
-public abstract class NNClassificationModel implements IgniteModel<Vector, Double>, Exportable<KNNModelFormat> {
+public abstract class NNClassificationModel implements IgniteModel<Vector, Double>, Exportable<KNNModelFormat>,
+ DeployableObject {
/** Amount of nearest neighbors. */
protected int k = 5;
@@ -229,4 +231,9 @@ public abstract class NNClassificationModel implements IgniteModel<Vector, Doubl
/** {@inheritDoc} */
@Override public abstract <P> void saveModel(Exporter<KNNModelFormat, P> exporter, P path);
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(distanceMeasure);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationModel.java
index b9d7270..14f2943 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationModel.java
@@ -35,7 +35,7 @@ import org.jetbrains.annotations.NotNull;
/**
* ANN model to predict labels in multi-class classification task.
*/
-public class ANNClassificationModel extends NNClassificationModel {
+public final class ANNClassificationModel extends NNClassificationModel {
/** */
private static final long serialVersionUID = -127312378991350345L;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationTrainer.java
index e547635..2219222 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationTrainer.java
@@ -68,7 +68,7 @@ public class ANNClassificationTrainer extends SingleLabelDatasetTrainer<ANNClass
* @param extractor Mapping from upstream entry to {@link LabeledVector}.
* @return Model.
*/
- @Override public <K, V> ANNClassificationModel fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> ANNClassificationModel fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> extractor) {
return updateModel(null, datasetBuilder, extractor);
@@ -175,7 +175,8 @@ public class ANNClassificationTrainer extends SingleLabelDatasetTrainer<ANNClass
try (Dataset<EmptyContext, LabeledVectorSet<LabeledVector>> dataset = datasetBuilder.build(
envBuilder,
(env, upstream, upstreamSize) -> new EmptyContext(),
- partDataBuilder
+ partDataBuilder,
+ learningEnvironment()
)) {
return dataset.compute(data -> {
CentroidStat res = new CentroidStat();
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/KNNUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/KNNUtils.java
index 64962be..5fdc2e1 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/KNNUtils.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/KNNUtils.java
@@ -22,6 +22,7 @@ import org.apache.ignite.ml.dataset.Dataset;
import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.dataset.PartitionDataBuilder;
import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.environment.LearningEnvironment;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
import org.apache.ignite.ml.preprocessing.Preprocessor;
import org.apache.ignite.ml.structures.LabeledVector;
@@ -44,6 +45,9 @@ public class KNNUtils {
@Nullable public static <K, V, C extends Serializable> Dataset<EmptyContext, LabeledVectorSet<LabeledVector>> buildDataset(
LearningEnvironmentBuilder envBuilder,
DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> vectorizer) {
+ LearningEnvironment environment = envBuilder.buildForTrainer();
+ environment.initDeployingContext(vectorizer);
+
PartitionDataBuilder<K, V, EmptyContext, LabeledVectorSet<LabeledVector>> partDataBuilder
= new LabeledDatasetPartitionDataBuilderOnHeap<>(vectorizer);
@@ -53,7 +57,8 @@ public class KNNUtils {
dataset = datasetBuilder.build(
envBuilder,
(env, upstream, upstreamSize) -> new EmptyContext(),
- partDataBuilder
+ partDataBuilder,
+ environment
);
}
return dataset;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeap.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeap.java
index e0376b8..62aae66 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeap.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeap.java
@@ -23,6 +23,7 @@ import org.apache.ignite.ml.dataset.Dataset;
import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.dataset.PartitionDataBuilder;
import org.apache.ignite.ml.dataset.primitive.data.SimpleLabeledDatasetData;
+import org.apache.ignite.ml.environment.LearningEnvironment;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
/**
@@ -41,11 +42,13 @@ public class LSQROnHeap<K, V> extends AbstractLSQR implements AutoCloseable {
*/
public LSQROnHeap(DatasetBuilder<K, V> datasetBuilder,
LearningEnvironmentBuilder envBuilder,
- PartitionDataBuilder<K, V, LSQRPartitionContext, SimpleLabeledDatasetData> partDataBuilder) {
+ PartitionDataBuilder<K, V, LSQRPartitionContext, SimpleLabeledDatasetData> partDataBuilder,
+ LearningEnvironment localLearningEnv) {
this.dataset = datasetBuilder.build(
envBuilder,
(env, upstream, upstreamSize) -> new LSQRPartitionContext(),
- partDataBuilder
+ partDataBuilder,
+ localLearningEnv
);
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/multiclass/MultiClassModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/multiclass/MultiClassModel.java
index a1ef652..a0a3cd4 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/multiclass/MultiClassModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/multiclass/MultiClassModel.java
@@ -26,7 +26,7 @@ import java.io.Serializable;
import java.util.*;
/** Base class for multi-classification model for set of classifiers. */
-public class MultiClassModel<M extends IgniteModel<Vector, Double>> implements IgniteModel<Vector, Double>, Exportable<MultiClassModel>, Serializable {
+public final class MultiClassModel<M extends IgniteModel<Vector, Double>> implements IgniteModel<Vector, Double>, Exportable<MultiClassModel>, Serializable {
/** */
private static final long serialVersionUID = -114986533359917L;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/multiclass/OneVsRestTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/multiclass/OneVsRestTrainer.java
index dee94d7..1e28ad3 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/multiclass/OneVsRestTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/multiclass/OneVsRestTrainer.java
@@ -56,7 +56,7 @@ public class OneVsRestTrainer<M extends IgniteModel<Vector, Double>>
}
/** {@inheritDoc} */
- @Override public <K, V> MultiClassModel<M> fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> MultiClassModel<M> fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> extractor) {
return updateModel(null, datasetBuilder, extractor);
@@ -111,7 +111,7 @@ public class OneVsRestTrainer<M extends IgniteModel<Vector, Double>>
try (Dataset<EmptyContext, LabelPartitionDataOnHeap> dataset = datasetBuilder.build(
envBuilder,
(env, upstream, upstreamSize) -> new EmptyContext(),
- partDataBuilder
+ partDataBuilder, learningEnvironment()
)) {
final Set<Double> clsLabels = dataset.compute(data -> {
final Set<Double> locClsLabels = new HashSet<>();
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/compound/CompoundNaiveBayesModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/compound/CompoundNaiveBayesModel.java
index 5daebb7..15b5213 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/compound/CompoundNaiveBayesModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/compound/CompoundNaiveBayesModel.java
@@ -18,12 +18,15 @@
package org.apache.ignite.ml.naivebayes.compound;
import java.io.Serializable;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import org.apache.ignite.ml.Exportable;
import org.apache.ignite.ml.Exporter;
import org.apache.ignite.ml.IgniteModel;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.naivebayes.discrete.DiscreteNaiveBayesModel;
@@ -33,7 +36,7 @@ import org.apache.ignite.ml.naivebayes.gaussian.GaussianNaiveBayesModel;
* A compound Naive Bayes model which uses a composition of{@code GaussianNaiveBayesModel} and {@code
* DiscreteNaiveBayesModel}.
*/
-public class CompoundNaiveBayesModel implements IgniteModel<Vector, Double>, Exportable<CompoundNaiveBayesModel>, Serializable {
+public class CompoundNaiveBayesModel implements IgniteModel<Vector, Double>, Exportable<CompoundNaiveBayesModel>, Serializable, DeployableObject {
/** Serial version uid. */
private static final long serialVersionUID = -5045925321135798960L;
@@ -156,4 +159,9 @@ public class CompoundNaiveBayesModel implements IgniteModel<Vector, Double>, Exp
}
return VectorUtils.of(newFeaturesValues);
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Arrays.asList(discreteModel, gaussianModel);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/compound/CompoundNaiveBayesTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/compound/CompoundNaiveBayesTrainer.java
index 9b8cd46..822f1c4 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/compound/CompoundNaiveBayesTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/compound/CompoundNaiveBayesTrainer.java
@@ -56,7 +56,7 @@ public class CompoundNaiveBayesTrainer extends SingleLabelDatasetTrainer<Compoun
private Collection<Integer> discreteFeatureIdsToSkip = Collections.emptyList();
/** {@inheritDoc} */
- @Override public <K, V> CompoundNaiveBayesModel fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> CompoundNaiveBayesModel fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> extractor) {
return updateModel(null, datasetBuilder, extractor);
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/discrete/DiscreteNaiveBayesModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/discrete/DiscreteNaiveBayesModel.java
index 44d0767..2e5aa71 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/discrete/DiscreteNaiveBayesModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/discrete/DiscreteNaiveBayesModel.java
@@ -18,15 +18,19 @@
package org.apache.ignite.ml.naivebayes.discrete;
import org.apache.ignite.ml.Exporter;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.naivebayes.BayesModel;
+import java.util.Collections;
+import java.util.List;
+
/**
* Discrete naive Bayes model which predicts result value {@code y} belongs to a class {@code C_k, k in [0..K]} as
* {@code p(C_k,y) =x_1*p_k1^x *...*x_i*p_ki^x_i}. Where {@code x_i} is a discrete feature, {@code p_ki} is a prior
* probability probability of class {@code p(x|C_k)}. Returns the number of the most possible class.
*/
-public class DiscreteNaiveBayesModel implements BayesModel<DiscreteNaiveBayesModel, Vector, Double> {
+public class DiscreteNaiveBayesModel implements BayesModel<DiscreteNaiveBayesModel, Vector, Double>, DeployableObject {
/** Serial version uid. */
private static final long serialVersionUID = -127386523291350345L;
@@ -142,4 +146,9 @@ public class DiscreteNaiveBayesModel implements BayesModel<DiscreteNaiveBayesMod
return thresholds.length;
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.emptyList();
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/discrete/DiscreteNaiveBayesTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/discrete/DiscreteNaiveBayesTrainer.java
index a80c402..339e128 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/discrete/DiscreteNaiveBayesTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/discrete/DiscreteNaiveBayesTrainer.java
@@ -50,7 +50,7 @@ public class DiscreteNaiveBayesTrainer extends SingleLabelDatasetTrainer<Discret
private double[][] bucketThresholds;
/** {@inheritDoc} */
- @Override public <K, V> DiscreteNaiveBayesModel fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> DiscreteNaiveBayesModel fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> extractor) {
return updateModel(null, datasetBuilder, extractor);
}
@@ -112,7 +112,7 @@ public class DiscreteNaiveBayesTrainer extends SingleLabelDatasetTrainer<Discret
}
}
return res;
- })) {
+ }, learningEnvironment())) {
DiscreteNaiveBayesSumsHolder sumsHolder = dataset.compute(t -> t, (a, b) -> {
if (a == null)
return b;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/gaussian/GaussianNaiveBayesModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/gaussian/GaussianNaiveBayesModel.java
index 29b8cc2..83db07d 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/gaussian/GaussianNaiveBayesModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/gaussian/GaussianNaiveBayesModel.java
@@ -18,14 +18,18 @@
package org.apache.ignite.ml.naivebayes.gaussian;
import org.apache.ignite.ml.Exporter;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.naivebayes.BayesModel;
+import java.util.Collections;
+import java.util.List;
+
/**
* Simple naive Bayes model which predicts result value {@code y} belongs to a class {@code C_k, k in [0..K]} as {@code
* p(C_k,y) = p(C_k)*p(y_1,C_k) *...*p(y_n,C_k) / p(y)}. Return the number of the most possible class.
*/
-public class GaussianNaiveBayesModel implements BayesModel<GaussianNaiveBayesModel, Vector, Double> {
+public class GaussianNaiveBayesModel implements BayesModel<GaussianNaiveBayesModel, Vector, Double>, DeployableObject {
/** Serial version uid. */
private static final long serialVersionUID = -127386523291350345L;
@@ -124,4 +128,9 @@ public class GaussianNaiveBayesModel implements BayesModel<GaussianNaiveBayesMod
private static double gauss(double x, double mean, double variance) {
return Math.exp(-1. * Math.pow(x - mean, 2) / (2. * variance)) / Math.sqrt(2. * Math.PI * variance);
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.emptyList();
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/gaussian/GaussianNaiveBayesTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/gaussian/GaussianNaiveBayesTrainer.java
index 84a16ad..f281aba 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/gaussian/GaussianNaiveBayesTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/naivebayes/gaussian/GaussianNaiveBayesTrainer.java
@@ -43,7 +43,7 @@ public class GaussianNaiveBayesTrainer extends SingleLabelDatasetTrainer<Gaussia
private boolean equiprobableClasses;
/** {@inheritDoc} */
- @Override public <K, V> GaussianNaiveBayesModel fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> GaussianNaiveBayesModel fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> extractor) {
return updateModel(null, datasetBuilder, extractor);
}
@@ -102,7 +102,7 @@ public class GaussianNaiveBayesTrainer extends SingleLabelDatasetTrainer<Gaussia
}
}
return res;
- }
+ }, learningEnvironment()
)) {
GaussianNaiveBayesSumsHolder sumsHolder = dataset.compute(t -> t, (a, b) -> {
if (a == null)
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
index fbcc850..7043fab 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
@@ -109,7 +109,7 @@ public class MLPTrainer<P extends Serializable> extends MultiLabelDatasetTrainer
}
/** {@inheritDoc} */
- @Override public <K, V> MultilayerPerceptron fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> MultilayerPerceptron fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> extractor) {
return updateModel(null, datasetBuilder, extractor);
}
@@ -126,7 +126,8 @@ public class MLPTrainer<P extends Serializable> extends MultiLabelDatasetTrainer
try (Dataset<EmptyContext, SimpleLabeledDatasetData> dataset = datasetBuilder.build(
envBuilder,
new EmptyContextBuilder<>(),
- new SimpleLabeledDatasetDataBuilder<>(extractor)
+ new SimpleLabeledDatasetDataBuilder<>(extractor),
+ learningEnvironment()
)) {
MultilayerPerceptron mdl;
if (lastLearnedMdl != null)
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java
index 4e17480..23809a6 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java
@@ -43,7 +43,7 @@ import static org.apache.ignite.ml.math.util.MatrixUtil.elementWiseTimes;
/**
* Class encapsulating logic of multilayer perceptron.
*/
-public class MultilayerPerceptron implements IgniteModel<Matrix, Matrix>, SmoothParametrized<MultilayerPerceptron>,
+public final class MultilayerPerceptron implements IgniteModel<Matrix, Matrix>, SmoothParametrized<MultilayerPerceptron>,
Serializable {
/**
* This MLP architecture.
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/pipeline/Pipeline.java b/modules/ml/src/main/java/org/apache/ignite/ml/pipeline/Pipeline.java
index 1060efd..b869aed 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/pipeline/Pipeline.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/pipeline/Pipeline.java
@@ -28,6 +28,7 @@ import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.environment.LearningEnvironment;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.preprocessing.PreprocessingTrainer;
@@ -137,7 +138,6 @@ public class Pipeline<K, V, C extends Serializable, L> {
finalPreprocessor = vectorizer;
preprocessingTrainers.forEach(e -> {
-
finalPreprocessor = e.fit(
envBuilder,
datasetBuilder,
@@ -145,10 +145,13 @@ public class Pipeline<K, V, C extends Serializable, L> {
);
});
+ LearningEnvironment env = LearningEnvironmentBuilder.defaultBuilder().buildForTrainer();
+ env.initDeployingContext(finalPreprocessor);
IgniteModel<Vector, Double> internalMdl = finalStage
.fit(
datasetBuilder,
- finalPreprocessor
+ finalPreprocessor,
+ env
);
return new PipelineMdl<K, V>()
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/pipeline/PipelineMdl.java b/modules/ml/src/main/java/org/apache/ignite/ml/pipeline/PipelineMdl.java
index 543dba7..a65687d 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/pipeline/PipelineMdl.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/pipeline/PipelineMdl.java
@@ -17,7 +17,10 @@
package org.apache.ignite.ml.pipeline;
+import java.util.Collections;
+import java.util.List;
import org.apache.ignite.ml.IgniteModel;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.preprocessing.Preprocessor;
@@ -27,7 +30,7 @@ import org.apache.ignite.ml.preprocessing.Preprocessor;
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
*/
-public class PipelineMdl<K, V> implements IgniteModel<Vector, Double> {
+public final class PipelineMdl<K, V> implements IgniteModel<Vector, Double>, DeployableObject {
/** Internal model produced by {@link Pipeline}. */
private IgniteModel<Vector, Double> internalMdl;
@@ -67,4 +70,9 @@ public class PipelineMdl<K, V> implements IgniteModel<Vector, Double> {
"internalMdl=" + internalMdl +
'}';
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(preprocessor);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java
index e66b546..dc50f46 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java
@@ -23,6 +23,7 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.environment.LearningEnvironment;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
/**
@@ -132,4 +133,18 @@ public interface PreprocessingTrainer<K, V> {
basePreprocessor
);
}
+
+ /**
+ * Returns local learning environment with initialized deploying context by base preptocessor.
+ *
+ * @param basePreprocessor Preprocessor.
+ * @param <K> Type of key.
+ * @param <V> Type of value.
+ * @return Learning environment.
+ */
+ public default <K,V> LearningEnvironment learningEnvironment(Preprocessor<K,V> basePreprocessor) {
+ LearningEnvironment env = LearningEnvironmentBuilder.defaultBuilder().buildForTrainer();
+ env.initDeployingContext(basePreprocessor);
+ return env;
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationPreprocessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationPreprocessor.java
index 26c7315..0d14e6b 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationPreprocessor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationPreprocessor.java
@@ -17,6 +17,9 @@
package org.apache.ignite.ml.preprocessing.binarization;
+import java.util.Collections;
+import java.util.List;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.preprocessing.Preprocessor;
import org.apache.ignite.ml.structures.LabeledVector;
@@ -29,7 +32,7 @@ import org.apache.ignite.ml.structures.LabeledVector;
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
*/
-public class BinarizationPreprocessor<K, V> implements Preprocessor<K, V> {
+public final class BinarizationPreprocessor<K, V> implements Preprocessor<K, V>, DeployableObject {
/** */
private static final long serialVersionUID = 6877811577892621239L;
@@ -72,4 +75,9 @@ public class BinarizationPreprocessor<K, V> implements Preprocessor<K, V> {
public double getThreshold() {
return threshold;
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(basePreprocessor);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/developer/MappedPreprocessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/developer/MappedPreprocessor.java
index 7cf65b5..713817b 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/developer/MappedPreprocessor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/developer/MappedPreprocessor.java
@@ -17,6 +17,9 @@
package org.apache.ignite.ml.preprocessing.developer;
+import java.util.Collections;
+import java.util.List;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.ml.preprocessing.Preprocessor;
import org.apache.ignite.ml.structures.LabeledVector;
@@ -29,7 +32,7 @@ import org.apache.ignite.ml.structures.LabeledVector;
* @param <L0> Type of original label.
* @param <L1> Type of mapped label.
*/
-public class MappedPreprocessor<K, V, L0, L1> implements Preprocessor<K, V> {
+public final class MappedPreprocessor<K, V, L0, L1> implements Preprocessor<K, V>, DeployableObject {
/** Original preprocessor. */
protected final Preprocessor<K, V> original;
@@ -51,4 +54,9 @@ public class MappedPreprocessor<K, V, L0, L1> implements Preprocessor<K, V> {
LabeledVector<L0> origVec = original.apply(key, value);
return mapping.apply(origVec);
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(original);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/developer/PatchedPreprocessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/developer/PatchedPreprocessor.java
index 6819199..82654a9 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/developer/PatchedPreprocessor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/developer/PatchedPreprocessor.java
@@ -17,6 +17,9 @@
package org.apache.ignite.ml.preprocessing.developer;
+import java.util.Collections;
+import java.util.List;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.ml.preprocessing.Preprocessor;
import org.apache.ignite.ml.structures.LabeledVector;
@@ -32,7 +35,7 @@ import org.apache.ignite.ml.structures.LabeledVector;
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
*/
-public class PatchedPreprocessor<K, V, L1, L2> implements Preprocessor<K, V> {
+public class PatchedPreprocessor<K, V, L1, L2> implements Preprocessor<K, V>, DeployableObject {
/** */
private static final long serialVersionUID = 6865823577892621239L;
@@ -65,4 +68,8 @@ public class PatchedPreprocessor<K, V, L1, L2> implements Preprocessor<K, V> {
return lbPatcher.apply(tmp);
}
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(basePreprocessor);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderPreprocessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderPreprocessor.java
index c3296d9..5e32ac3 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderPreprocessor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderPreprocessor.java
@@ -46,10 +46,10 @@ public abstract class EncoderPreprocessor<K, V> implements Preprocessor<K, V> {
* Constructs a new instance of String Encoder preprocessor.
*
* @param basePreprocessor Base preprocessor.
- * @param handledIndices Handled indices.
+ * @param handledIndices Handled indices.
*/
public EncoderPreprocessor(Map<String, Integer>[] encodingValues,
- Preprocessor<K, V> basePreprocessor, Set<Integer> handledIndices) {
+ Preprocessor<K, V> basePreprocessor, Set<Integer> handledIndices) {
this.handledIndices = handledIndices;
this.encodingValues = encodingValues;
this.basePreprocessor = basePreprocessor;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java
index 5703ea0..9b331d0 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java
@@ -76,7 +76,7 @@ public class EncoderTrainer<K, V> implements PreprocessingTrainer<K, V> {
}
return new EncoderPartitionData()
.withCategoryFrequencies(categoryFrequencies);
- }
+ }, learningEnvironment(basePreprocessor)
)) {
switch (encoderType) {
case ONE_HOT_ENCODER:
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/onehotencoder/OneHotEncoderPreprocessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/onehotencoder/OneHotEncoderPreprocessor.java
index 2f75e16..72fcdf02 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/onehotencoder/OneHotEncoderPreprocessor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/onehotencoder/OneHotEncoderPreprocessor.java
@@ -17,8 +17,11 @@
package org.apache.ignite.ml.preprocessing.encoding.onehotencoder;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.exceptions.preprocessing.UnknownCategorialFeatureValue;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.preprocessing.Preprocessor;
@@ -46,7 +49,7 @@ import org.apache.ignite.ml.structures.LabeledVector;
*
* NOTE: the index value associated with NULL will located in binary vector according the frequency of NULL values.
*/
-public class OneHotEncoderPreprocessor<K, V> extends EncoderPreprocessor<K, V> {
+public final class OneHotEncoderPreprocessor<K, V> extends EncoderPreprocessor<K, V> implements DeployableObject {
/** */
private static final long serialVersionUID = 6237812226552623469L;
@@ -147,4 +150,9 @@ public class OneHotEncoderPreprocessor<K, V> extends EncoderPreprocessor<K, V> {
return idxOff;
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(basePreprocessor);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/stringencoder/StringEncoderPreprocessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/stringencoder/StringEncoderPreprocessor.java
index 4c334a9..07c8442 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/stringencoder/StringEncoderPreprocessor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/stringencoder/StringEncoderPreprocessor.java
@@ -17,8 +17,11 @@
package org.apache.ignite.ml.preprocessing.encoding.stringencoder;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.exceptions.preprocessing.UnknownCategorialFeatureValue;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.preprocessing.Preprocessor;
@@ -45,7 +48,7 @@ import org.apache.ignite.ml.structures.LabeledVector;
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
*/
-public class StringEncoderPreprocessor<K, V> extends EncoderPreprocessor<K, V> {
+public final class StringEncoderPreprocessor<K, V> extends EncoderPreprocessor<K, V> implements DeployableObject {
/** */
protected static final long serialVersionUID = 6237712226382623488L;
@@ -85,4 +88,9 @@ public class StringEncoderPreprocessor<K, V> extends EncoderPreprocessor<K, V> {
}
return new LabeledVector(VectorUtils.of(res), tmp.label());
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(basePreprocessor);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/imputing/ImputerPreprocessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/imputing/ImputerPreprocessor.java
index f23fde2..47c7f35 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/imputing/ImputerPreprocessor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/imputing/ImputerPreprocessor.java
@@ -17,6 +17,9 @@
package org.apache.ignite.ml.preprocessing.imputing;
+import java.util.Collections;
+import java.util.List;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.preprocessing.Preprocessor;
import org.apache.ignite.ml.structures.LabeledVector;
@@ -27,7 +30,7 @@ import org.apache.ignite.ml.structures.LabeledVector;
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
*/
-public class ImputerPreprocessor<K, V> implements Preprocessor<K, V> {
+public final class ImputerPreprocessor<K, V> implements Preprocessor<K, V>, DeployableObject {
/** */
private static final long serialVersionUID = 6887800576392623469L;
@@ -66,4 +69,9 @@ public class ImputerPreprocessor<K, V> implements Preprocessor<K, V> {
}
return res;
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(basePreprocessor);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/imputing/ImputerTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/imputing/ImputerTrainer.java
index cb7f551..63245cf 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/imputing/ImputerTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/imputing/ImputerTrainer.java
@@ -85,7 +85,7 @@ public class ImputerTrainer<K, V> implements PreprocessingTrainer<K, V> {
default: throw new UnsupportedOperationException("The chosen strategy is not supported");
}
return partData;
- }
+ }, learningEnvironment(basePreprocessor)
)) {
Vector imputingValues;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/maxabsscaling/MaxAbsScalerPreprocessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/maxabsscaling/MaxAbsScalerPreprocessor.java
index aa8a49f..01ccead 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/maxabsscaling/MaxAbsScalerPreprocessor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/maxabsscaling/MaxAbsScalerPreprocessor.java
@@ -17,6 +17,9 @@
package org.apache.ignite.ml.preprocessing.maxabsscaling;
+import java.util.Collections;
+import java.util.List;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.preprocessing.Preprocessor;
import org.apache.ignite.ml.structures.LabeledVector;
@@ -31,7 +34,7 @@ import org.apache.ignite.ml.structures.LabeledVector;
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
*/
-public class MaxAbsScalerPreprocessor<K, V> implements Preprocessor<K, V> {
+public final class MaxAbsScalerPreprocessor<K, V> implements Preprocessor<K, V>, DeployableObject {
/** */
private static final long serialVersionUID = 1L;
@@ -74,4 +77,9 @@ public class MaxAbsScalerPreprocessor<K, V> implements Preprocessor<K, V> {
public double[] getMaxAbs() {
return maxAbs;
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(basePreprocessor);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/maxabsscaling/MaxAbsScalerTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/maxabsscaling/MaxAbsScalerTrainer.java
index 1e7608d..b7678f9 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/maxabsscaling/MaxAbsScalerTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/maxabsscaling/MaxAbsScalerTrainer.java
@@ -63,7 +63,7 @@ public class MaxAbsScalerTrainer<K, V> implements PreprocessingTrainer<K, V> {
}
}
return new MaxAbsScalerPartitionData(maxAbs);
- }
+ }, learningEnvironment(basePreprocessor)
)) {
double[] maxAbs = dataset.compute(MaxAbsScalerPartitionData::getMaxAbs,
(a, b) -> {
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/minmaxscaling/MinMaxScalerPreprocessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/minmaxscaling/MinMaxScalerPreprocessor.java
index 79c70d1..e5372d5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/minmaxscaling/MinMaxScalerPreprocessor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/minmaxscaling/MinMaxScalerPreprocessor.java
@@ -17,6 +17,9 @@
package org.apache.ignite.ml.preprocessing.minmaxscaling;
+import java.util.Collections;
+import java.util.List;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.preprocessing.Preprocessor;
import org.apache.ignite.ml.structures.LabeledVector;
@@ -32,7 +35,7 @@ import org.apache.ignite.ml.structures.LabeledVector;
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
*/
-public class MinMaxScalerPreprocessor<K, V> implements Preprocessor<K, V> {
+public final class MinMaxScalerPreprocessor<K, V> implements Preprocessor<K, V>, DeployableObject {
/** */
private static final long serialVersionUID = 6997800576392623469L;
@@ -94,4 +97,9 @@ public class MinMaxScalerPreprocessor<K, V> implements Preprocessor<K, V> {
public double[] getMax() {
return max;
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(basePreprocessor);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/minmaxscaling/MinMaxScalerTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/minmaxscaling/MinMaxScalerTrainer.java
index cca2e30..54a6d59 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/minmaxscaling/MinMaxScalerTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/minmaxscaling/MinMaxScalerTrainer.java
@@ -78,7 +78,7 @@ public class MinMaxScalerTrainer<K, V> implements PreprocessingTrainer<K, V> {
}
return new MinMaxScalerPartitionData(min, max);
- }
+ }, learningEnvironment(basePreprocessor)
)) {
double[][] minMax = dataset.compute(
data -> data.getMin() != null ? new double[][]{ data.getMin(), data.getMax() } : null,
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationPreprocessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationPreprocessor.java
index fb67865..4dd969e 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationPreprocessor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationPreprocessor.java
@@ -17,6 +17,9 @@
package org.apache.ignite.ml.preprocessing.normalization;
+import java.util.Collections;
+import java.util.List;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.math.functions.Functions;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.functions.IgniteDoubleFunction;
@@ -34,7 +37,7 @@ import org.apache.ignite.ml.structures.LabeledVector;
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
*/
-public class NormalizationPreprocessor<K, V> implements Preprocessor<K, V> {
+public final class NormalizationPreprocessor<K, V> implements Preprocessor<K, V>, DeployableObject {
/** */
private static final long serialVersionUID = 6873438115778921295L;
@@ -93,4 +96,9 @@ public class NormalizationPreprocessor<K, V> implements Preprocessor<K, V> {
public double p() {
return p;
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(basePreprocessor);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/standardscaling/StandardScalerPreprocessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/standardscaling/StandardScalerPreprocessor.java
index c060448..1eb2e10 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/standardscaling/StandardScalerPreprocessor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/standardscaling/StandardScalerPreprocessor.java
@@ -17,6 +17,9 @@
package org.apache.ignite.ml.preprocessing.standardscaling;
+import java.util.Collections;
+import java.util.List;
+import org.apache.ignite.ml.environment.deploy.DeployableObject;
import org.apache.ignite.ml.preprocessing.Preprocessor;
import org.apache.ignite.ml.structures.LabeledVector;
@@ -33,7 +36,7 @@ import org.apache.ignite.ml.structures.LabeledVector;
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
*/
-public class StandardScalerPreprocessor<K, V> implements Preprocessor<K, V> {
+public final class StandardScalerPreprocessor<K, V> implements Preprocessor<K, V>, DeployableObject {
/** */
private static final long serialVersionUID = -5977957318991608203L;
@@ -89,4 +92,9 @@ public class StandardScalerPreprocessor<K, V> implements Preprocessor<K, V> {
public double[] getSigmas() {
return sigmas;
}
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(basePreprocessor);
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/standardscaling/StandardScalerTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/standardscaling/StandardScalerTrainer.java
index fd5b967..4d0a8c7 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/standardscaling/StandardScalerTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/standardscaling/StandardScalerTrainer.java
@@ -85,7 +85,7 @@ public class StandardScalerTrainer<K, V> implements PreprocessingTrainer<K, V> {
}
}
return new StandardScalerData(sum, squaredSum, cnt);
- }
+ }, learningEnvironment(basePreprocessor)
)) {
return dataset.compute(data -> data,
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
index 280a15a..9ecc257 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
@@ -38,7 +38,7 @@ import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
*/
public class LinearRegressionLSQRTrainer extends SingleLabelDatasetTrainer<LinearRegressionModel> {
/** {@inheritDoc} */
- @Override public <K, V> LinearRegressionModel fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> LinearRegressionModel fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> extractor) {
return updateModel(null, datasetBuilder, extractor);
@@ -61,14 +61,14 @@ public class LinearRegressionLSQRTrainer extends SingleLabelDatasetTrainer<Linea
@Override protected <K, V> LinearRegressionModel updateModel(LinearRegressionModel mdl,
DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> extractor) {
-
LSQRResult res;
PatchedPreprocessor<K, V, Double, double[]> patchedPreprocessor = new PatchedPreprocessor<>(LinearRegressionLSQRTrainer::extendLabeledVector, extractor);
try (LSQROnHeap<K, V> lsqr = new LSQROnHeap<>(
datasetBuilder, envBuilder,
- new SimpleLabeledDatasetDataBuilder<>(patchedPreprocessor))) {
+ new SimpleLabeledDatasetDataBuilder<>(patchedPreprocessor),
+ learningEnvironment())) {
double[] x0 = null;
if (mdl != null) {
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionModel.java
index b298524..829076d 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionModel.java
@@ -28,7 +28,7 @@ import org.apache.ignite.ml.math.primitives.vector.Vector;
* Simple linear regression model which predicts result value Y as a linear combination of input variables:
* Y = weights * X + intercept.
*/
-public class LinearRegressionModel implements IgniteModel<Vector, Double>, Exportable<LinearRegressionModel>, Serializable {
+public final class LinearRegressionModel implements IgniteModel<Vector, Double>, Exportable<LinearRegressionModel>, Serializable {
/** */
private static final long serialVersionUID = -105984600091550226L;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
index c8b9c58..da813fc 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
@@ -84,7 +84,7 @@ public class LinearRegressionSGDTrainer<P extends Serializable> extends SingleLa
}
/** {@inheritDoc} */
- @Override public <K, V> LinearRegressionModel fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> LinearRegressionModel fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> extractor) {
return updateModel(null, datasetBuilder, extractor);
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionModel.java
index f041fbf..c96b5bf 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionModel.java
@@ -27,7 +27,7 @@ import org.apache.ignite.ml.math.primitives.vector.Vector;
/**
* Logistic regression (logit model) is a generalized linear model used for binomial regression.
*/
-public class LogisticRegressionModel implements IgniteModel<Vector, Double>, Exportable<LogisticRegressionModel>, Serializable {
+public final class LogisticRegressionModel implements IgniteModel<Vector, Double>, Exportable<LogisticRegressionModel>, Serializable {
/** */
private static final long serialVersionUID = -133984600091550776L;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionSGDTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionSGDTrainer.java
index 63dfb95..c0404db 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionSGDTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionSGDTrainer.java
@@ -63,7 +63,7 @@ public class LogisticRegressionSGDTrainer extends SingleLabelDatasetTrainer<Logi
private long seed = 1234L;
/** {@inheritDoc} */
- @Override public <K, V> LogisticRegressionModel fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> LogisticRegressionModel fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> extractor) {
return updateModel(null, datasetBuilder, extractor);
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/sql/SqlDatasetBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/sql/SqlDatasetBuilder.java
index 4d31ca3..776669a 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/sql/SqlDatasetBuilder.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/sql/SqlDatasetBuilder.java
@@ -59,6 +59,6 @@ public class SqlDatasetBuilder extends CacheBasedDatasetBuilder<Object, BinaryOb
*/
public SqlDatasetBuilder(Ignite ignite, String upstreamCache, IgniteBiPredicate<Object, BinaryObject> filter,
UpstreamTransformerBuilder transformerBuilder) {
- super(ignite, ignite.cache(upstreamCache), filter, transformerBuilder, true);
+ super(ignite, ignite.cache(upstreamCache), filter, transformerBuilder, true, CacheBasedDatasetBuilder.DEFAULT_NUMBER_OF_RETRIES);
}
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearClassificationModel.java
index 4ee822b..57006c1 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearClassificationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearClassificationModel.java
@@ -27,7 +27,7 @@ import org.apache.ignite.ml.math.primitives.vector.Vector;
/**
* Base class for SVM linear classification model.
*/
-public class SVMLinearClassificationModel implements IgniteModel<Vector, Double>, Exportable<SVMLinearClassificationModel>, Serializable {
+public final class SVMLinearClassificationModel implements IgniteModel<Vector, Double>, Exportable<SVMLinearClassificationModel>, Serializable {
/** */
private static final long serialVersionUID = -996984622291440226L;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearClassificationTrainer.java
index 8ce5004..5c53fff 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearClassificationTrainer.java
@@ -61,7 +61,7 @@ public class SVMLinearClassificationTrainer extends SingleLabelDatasetTrainer<SV
* @param preprocessor Extractor of {@link UpstreamEntry} into {@link LabeledVector}.
* @return Model.
*/
- @Override public <K, V> SVMLinearClassificationModel fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> SVMLinearClassificationModel fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> preprocessor) {
return updateModel(null, datasetBuilder, preprocessor);
@@ -93,7 +93,8 @@ public class SVMLinearClassificationTrainer extends SingleLabelDatasetTrainer<SV
try (Dataset<EmptyContext, LabeledVectorSet<LabeledVector>> dataset = datasetBuilder.build(
envBuilder,
(env, upstream, upstreamSize) -> new EmptyContext(),
- partDataBuilder
+ partDataBuilder,
+ learningEnvironment()
)) {
if (mdl == null) {
final int cols = dataset.compute(org.apache.ignite.ml.structures.Dataset::colSize, (a, b) -> {
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/AdaptableDatasetTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/AdaptableDatasetTrainer.java
index e4486e3..0946a56 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/AdaptableDatasetTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/AdaptableDatasetTrainer.java
@@ -93,7 +93,7 @@ public class AdaptableDatasetTrainer<I, O, IW, OW, M extends IgniteModel<IW, OW>
}
/** {@inheritDoc} */
- @Override public <K, V> AdaptableDatasetModel<I, O, IW, OW, M> fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> AdaptableDatasetModel<I, O, IW, OW, M> fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> extractor) {
M fit = wrapped.
withEnvironmentBuilder(envBuilder)
@@ -165,7 +165,7 @@ public class AdaptableDatasetTrainer<I, O, IW, OW, M extends IgniteModel<IW, OW>
public AdaptableDatasetTrainer<I, O, IW, OW, M, L> withDatasetMapping(DatasetMapping<L, L> mapping) {
return of(new DatasetTrainer<M, L>() {
/** {@inheritDoc} */
- @Override public <K, V> M fit(DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> extractor) {
+ @Override public <K, V> M fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> extractor) {
return wrapped.fit(datasetBuilder, extractor.map(lv -> new LabeledVector<>(
mapping.mapFeatures(lv.features()),
mapping.mapLabels((L)lv.label())
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java
index 8cef0f0..11a43d0 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java
@@ -59,7 +59,7 @@ public abstract class DatasetTrainer<M extends IgniteModel, L> {
public static <I, L> DatasetTrainer<IgniteModel<I, I>, L> identityTrainer() {
return new DatasetTrainer<IgniteModel<I, I>, L>() {
/** {@inheritDoc} */
- @Override public <K, V> IgniteModel<I, I> fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> IgniteModel<I, I> fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> preprocessor) {
return x -> x;
}
@@ -86,7 +86,38 @@ public abstract class DatasetTrainer<M extends IgniteModel, L> {
* @param <V> Type of a value in {@code upstream} data.
* @return Model.
*/
- public abstract <K, V> M fit(DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> preprocessor);
+ public <K, V> M fit(DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> preprocessor) {
+ learningEnvironment().initDeployingContext(preprocessor);
+
+ return fitWithInitializedDeployingContext(datasetBuilder, preprocessor);
+ }
+
+ /**
+ * Trains model based on the specified data.
+ *
+ * @param datasetBuilder Dataset builder.
+ * @param preprocessor Extractor of {@link UpstreamEntry} into {@link LabeledVector}.
+ * @param learningEnvironment Local learning environment.
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ * @return Model.
+ */
+ public <K, V> M fit(DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> preprocessor, LearningEnvironment learningEnvironment) {
+ environment = learningEnvironment;
+ return fitWithInitializedDeployingContext(datasetBuilder, preprocessor);
+ }
+
+ /**
+ * Trains model based on the specified data.
+ *
+ * @param datasetBuilder Dataset builder.
+ * @param preprocessor Extractor of {@link UpstreamEntry} into {@link LabeledVector}.
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ * @return Model.
+ */
+ protected abstract <K, V> M fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
+ Preprocessor<K, V> preprocessor);
/**
* Gets state of model in arguments, compare it with training parameters of trainer and if they are fit then trainer
@@ -102,9 +133,11 @@ public abstract class DatasetTrainer<M extends IgniteModel, L> {
public <K, V> M update(M mdl, DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> preprocessor) {
if (mdl != null) {
- if (isUpdateable(mdl))
+ if (isUpdateable(mdl)) {
+ learningEnvironment().initDeployingContext(preprocessor);
+
return updateModel(mdl, datasetBuilder, preprocessor);
- else {
+ } else {
environment.logger(getClass()).log(
MLLogger.VerboseLevel.HIGH,
"Model cannot be updated because of initial state of " +
@@ -350,7 +383,7 @@ public abstract class DatasetTrainer<M extends IgniteModel, L> {
return old.updateModel(mdl, datasetBuilder, getNewExtractor(preprocessor));
}
- @Override public <K, V> M fit(DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> preprocessor) {
+ @Override public <K, V> M fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> preprocessor) {
return old.fit(datasetBuilder, getNewExtractor(preprocessor));
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
index 15c5cd1..9b4a5ed 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
@@ -106,12 +106,13 @@ public abstract class DecisionTree<T extends ImpurityMeasure<T>> extends SingleL
}
/** {@inheritDoc} */
- @Override public <K, V> DecisionTreeNode fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> DecisionTreeNode fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> preprocessor) {
try (Dataset<EmptyContext, DecisionTreeData> dataset = datasetBuilder.build(
envBuilder,
new EmptyContextBuilder<>(),
- new DecisionTreeDataBuilder<>(preprocessor, usingIdx)
+ new DecisionTreeDataBuilder<>(preprocessor, usingIdx),
+ learningEnvironment()
)) {
return fit(dataset);
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java
index 7eff06c..e26eb6f 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java
@@ -22,7 +22,7 @@ import org.apache.ignite.ml.math.primitives.vector.Vector;
/**
* Decision tree conditional (non-leaf) node.
*/
-public class DecisionTreeConditionalNode implements DecisionTreeNode {
+public final class DecisionTreeConditionalNode implements DecisionTreeNode {
/** */
private static final long serialVersionUID = 981630737007982172L;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java
index 43f0f05..bde8e95 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java
@@ -22,7 +22,7 @@ import org.apache.ignite.ml.math.primitives.vector.Vector;
/**
* Decision tree leaf node which contains value.
*/
-public class DecisionTreeLeafNode implements DecisionTreeNode {
+public final class DecisionTreeLeafNode implements DecisionTreeNode {
/** */
private static final long serialVersionUID = -472145568088482206L;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBOnTreesLearningStrategy.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBOnTreesLearningStrategy.java
index 5cccd5b..1c25f73 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBOnTreesLearningStrategy.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBOnTreesLearningStrategy.java
@@ -29,6 +29,7 @@ import org.apache.ignite.ml.dataset.Dataset;
import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.dataset.primitive.builder.context.EmptyContextBuilder;
import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.environment.LearningEnvironment;
import org.apache.ignite.ml.environment.logging.MLLogger;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
@@ -59,6 +60,9 @@ public class GDBOnTreesLearningStrategy extends GDBLearningStrategy {
@Override public <K, V> List<IgniteModel<Vector, Double>> update(GDBTrainer.GDBModel mdlToUpdate,
DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> vectorizer) {
+ LearningEnvironment environment = envBuilder.buildForTrainer();
+ environment.initDeployingContext(vectorizer);
+
DatasetTrainer<? extends IgniteModel<Vector, Double>, Double> trainer = baseMdlTrainerBuilder.get();
assert trainer instanceof DecisionTree;
DecisionTree decisionTreeTrainer = (DecisionTree)trainer;
@@ -71,7 +75,8 @@ public class GDBOnTreesLearningStrategy extends GDBLearningStrategy {
try (Dataset<EmptyContext, DecisionTreeData> dataset = datasetBuilder.build(
envBuilder,
new EmptyContextBuilder<>(),
- new DecisionTreeDataBuilder<>(vectorizer, useIdx)
+ new DecisionTreeDataBuilder<>(vectorizer, useIdx),
+ environment
)) {
for (int i = 0; i < cntOfIterations; i++) {
double[] weights = Arrays.copyOf(compositionWeights, models.size());
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestTrainer.java
index c333990..2776422 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestTrainer.java
@@ -110,13 +110,14 @@ public abstract class RandomForestTrainer<L, S extends ImpurityComputer<Bootstra
}
/** {@inheritDoc} */
- @Override public <K, V> ModelsComposition fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> ModelsComposition fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> preprocessor) {
List<TreeRoot> models = null;
try (Dataset<EmptyContext, BootstrappedDatasetPartition> dataset = datasetBuilder.build(
envBuilder,
new EmptyContextBuilder<>(),
- new BootstrappedDatasetBuilder<>(preprocessor, amountOfTrees, subSampleSize))) {
+ new BootstrappedDatasetBuilder<>(preprocessor, amountOfTrees, subSampleSize),
+ learningEnvironment())) {
if (!init(dataset))
return buildComposition(Collections.emptyList());
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/TestUtils.java b/modules/ml/src/test/java/org/apache/ignite/ml/TestUtils.java
index 4203b14..ddea93b 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/TestUtils.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/TestUtils.java
@@ -454,7 +454,7 @@ public class TestUtils {
return ml;
}
- @Override public <K, V> M fit(DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> preprocessor) {
+ @Override public <K, V> M fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> preprocessor) {
return null;
}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/composition/bagging/BaggingTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/composition/bagging/BaggingTest.java
index 1fac7bc..fa7a7c9 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/composition/bagging/BaggingTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/composition/bagging/BaggingTest.java
@@ -183,13 +183,14 @@ public class BaggingTest extends TrainerTest {
}
/** {@inheritDoc} */
- @Override public <K, V> IgniteModel<Vector, Double> fit(
+ @Override public <K, V> IgniteModel<Vector, Double> fitWithInitializedDeployingContext(
DatasetBuilder<K, V> datasetBuilder,
Preprocessor<K, V> extractor) {
Dataset<Long, CountData> dataset = datasetBuilder.build(
TestUtils.testEnvBuilder(),
(env, upstreamData, upstreamDataSize) -> upstreamDataSize,
- (env, upstreamData, upstreamDataSize, ctx) -> new CountData(upstreamDataSize)
+ (env, upstreamData, upstreamDataSize, ctx) -> new CountData(upstreamDataSize),
+ TestUtils.testEnvBuilder().buildForTrainer()
);
Long cnt = dataset.computeWithCtx(cntr, BaggingTest::plusOfNullables);
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceCheckerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceCheckerTest.java
index 112fdfe..bd72535 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceCheckerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceCheckerTest.java
@@ -50,7 +50,8 @@ public class MeanAbsValueConvergenceCheckerTest extends ConvergenceCheckerTest {
try(LocalDataset<EmptyContext, FeatureMatrixWithLabelsOnHeapData> dataset = datasetBuilder.build(
envBuilder,
- new EmptyContextBuilder<>(), new FeatureMatrixWithLabelsOnHeapDataBuilder<>(vectorizer))) {
+ new EmptyContextBuilder<>(), new FeatureMatrixWithLabelsOnHeapDataBuilder<>(vectorizer),
+ envBuilder.buildForTrainer())) {
double onDSError = checker.computeMeanErrorOnDataset(dataset, notConvergedMdl);
Assert.assertEquals(1.55, onDSError, 0.01);
@@ -69,7 +70,8 @@ public class MeanAbsValueConvergenceCheckerTest extends ConvergenceCheckerTest {
try(LocalDataset<EmptyContext, FeatureMatrixWithLabelsOnHeapData> dataset = datasetBuilder.build(
TestUtils.testEnvBuilder(),
- new EmptyContextBuilder<>(), new FeatureMatrixWithLabelsOnHeapDataBuilder<>(vectorizer))) {
+ new EmptyContextBuilder<>(), new FeatureMatrixWithLabelsOnHeapDataBuilder<>(vectorizer),
+ TestUtils.testEnvBuilder().buildForTrainer())) {
double onDSError = checker.computeMeanErrorOnDataset(dataset, notConvergedMdl);
Assert.assertEquals(9090.41, onDSError, 0.01);
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceCheckerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceCheckerTest.java
index 3577e05..6784cde 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceCheckerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceCheckerTest.java
@@ -53,7 +53,8 @@ public class MedianOfMedianConvergenceCheckerTest extends ConvergenceCheckerTest
try(LocalDataset<EmptyContext, FeatureMatrixWithLabelsOnHeapData> dataset = datasetBuilder.build(
envBuilder,
- new EmptyContextBuilder<>(), new FeatureMatrixWithLabelsOnHeapDataBuilder<>(vectorizer))) {
+ new EmptyContextBuilder<>(), new FeatureMatrixWithLabelsOnHeapDataBuilder<>(vectorizer),
+ TestUtils.testEnvBuilder().buildForTrainer())) {
double onDSError = checker.computeMeanErrorOnDataset(dataset, notConvergedMdl);
Assert.assertEquals(1.6, onDSError, 0.01);
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetBuilderTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetBuilderTest.java
index 40cf4a7..7fb061d 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetBuilderTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetBuilderTest.java
@@ -67,7 +67,8 @@ public class CacheBasedDatasetBuilderTest extends GridCommonAbstractTest {
CacheBasedDataset<Integer, String, Long, AutoCloseable> dataset = builder.build(
TestUtils.testEnvBuilder(),
(env, upstream, upstreamSize) -> upstreamSize,
- (env, upstream, upstreamSize, ctx) -> null
+ (env, upstream, upstreamSize, ctx) -> null,
+ TestUtils.testEnvBuilder().buildForTrainer()
);
Affinity<Integer> upstreamAffinity = ignite.affinity(upstreamCache.getName());
@@ -120,7 +121,8 @@ public class CacheBasedDatasetBuilderTest extends GridCommonAbstractTest {
assertEquals(Integer.valueOf(2), entry.getValue());
assertFalse(upstream.hasNext());
return null;
- }
+ },
+ TestUtils.testEnvBuilder().buildForTrainer()
);
dataset.compute(data -> {});
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetTest.java
index cc25306..1a951a8 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetTest.java
@@ -88,7 +88,8 @@ public class CacheBasedDatasetTest extends GridCommonAbstractTest {
CacheBasedDataset<Integer, String, Long, SimpleDatasetData> dataset = builder.build(
TestUtils.testEnvBuilder(),
(env, upstream, upstreamSize) -> upstreamSize,
- (env, upstream, upstreamSize, ctx) -> new SimpleDatasetData(new double[0], 0)
+ (env, upstream, upstreamSize, ctx) -> new SimpleDatasetData(new double[0], 0),
+ TestUtils.testEnvBuilder().buildForTrainer()
);
assertEquals("Upstream cache name from dataset",
@@ -145,7 +146,8 @@ public class CacheBasedDatasetTest extends GridCommonAbstractTest {
CacheBasedDataset<Integer, String, Long, SimpleDatasetData> dataset = builder.build(
TestUtils.testEnvBuilder(),
(env, upstream, upstreamSize) -> upstreamSize,
- (env, upstream, upstreamSize, ctx) -> new SimpleDatasetData(new double[0], 0)
+ (env, upstream, upstreamSize, ctx) -> new SimpleDatasetData(new double[0], 0),
+ TestUtils.testEnvBuilder().buildForTrainer()
);
assertTrue("Before computation all partitions should not be reserved",
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/util/ComputeUtilsTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/util/ComputeUtilsTest.java
index 74f4c73..668860b 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/util/ComputeUtilsTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/util/ComputeUtilsTest.java
@@ -35,6 +35,7 @@ import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.ml.TestUtils;
import org.apache.ignite.ml.dataset.UpstreamEntry;
import org.apache.ignite.ml.dataset.UpstreamTransformerBuilder;
+import org.apache.ignite.ml.environment.deploy.DeployingContext;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Test;
@@ -89,7 +90,8 @@ public class ComputeUtilsTest extends GridCommonAbstractTest {
ignite,
Arrays.asList(firstCacheName, secondCacheName),
part -> part,
- 0
+ 0,
+ DeployingContext.unitialized()
);
}
catch (IllegalStateException expectedException) {
@@ -134,7 +136,7 @@ public class ComputeUtilsTest extends GridCommonAbstractTest {
cnt.incrementAndGet();
return part;
- }, 0);
+ }, 0, DeployingContext.unitialized());
assertEquals(1, cnt.get());
}
@@ -193,7 +195,8 @@ public class ComputeUtilsTest extends GridCommonAbstractTest {
TestUtils.testEnvBuilder().buildForWorker(part),
false
),
- 0
+ 0,
+ DeployingContext.unitialized()
);
assertEquals(1, data.size());
@@ -243,7 +246,8 @@ public class ComputeUtilsTest extends GridCommonAbstractTest {
TestUtils.testEnvBuilder(),
0,
0,
- false
+ false,
+ DeployingContext.unitialized()
);
assertEquals(1, datasetCache.size());
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/local/LocalDatasetBuilderTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/local/LocalDatasetBuilderTest.java
index 6088140..6d647d6 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/local/LocalDatasetBuilderTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/local/LocalDatasetBuilderTest.java
@@ -108,7 +108,8 @@ public class LocalDatasetBuilderTest {
return builder.build(
TestUtils.testEnvBuilder(),
partCtxBuilder.andThen(x -> null),
- partDataBuilder.andThen((x, y) -> x)
+ partDataBuilder.andThen((x, y) -> x),
+ TestUtils.testEnvBuilder().buildForTrainer()
);
}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/environment/EnvironmentTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/environment/EnvironmentTestSuite.java
index ae9f490..cdbd935 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/environment/EnvironmentTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/environment/EnvironmentTestSuite.java
@@ -17,6 +17,8 @@
package org.apache.ignite.ml.environment;
+import org.apache.ignite.ml.environment.deploy.DeployingContextImplTest;
+import org.apache.ignite.ml.environment.deploy.MLDeployingTest;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@@ -27,7 +29,9 @@ import org.junit.runners.Suite;
@Suite.SuiteClasses({
LearningEnvironmentBuilderTest.class,
LearningEnvironmentTest.class,
- PromiseTest.class
+ PromiseTest.class,
+ DeployingContextImplTest.class,
+ MLDeployingTest.class
})
public class EnvironmentTestSuite {
}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/environment/LearningEnvironmentTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/environment/LearningEnvironmentTest.java
index cc27f1d..c8e50b4 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/environment/LearningEnvironmentTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/environment/LearningEnvironmentTest.java
@@ -87,11 +87,12 @@ public class LearningEnvironmentTest {
DatasetTrainer<IgniteModel<Object, Vector>, Void> trainer = new DatasetTrainer<IgniteModel<Object, Vector>, Void>() {
/** {@inheritDoc} */
- @Override public <K, V> IgniteModel<Object, Vector> fit(DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> preprocessor) {
+ @Override public <K, V> IgniteModel<Object, Vector> fitWithInitializedDeployingContext(DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> preprocessor) {
Dataset<EmptyContext, TestUtils.DataWrapper<Integer>> ds = datasetBuilder.build(envBuilder,
new EmptyContextBuilder<>(),
(PartitionDataBuilder<K, V, EmptyContext, TestUtils.DataWrapper<Integer>>)(env, upstreamData, upstreamDataSize, ctx) ->
- TestUtils.DataWrapper.of(env.partition()));
+ TestUtils.DataWrapper.of(env.partition()),
+ envBuilder.buildForTrainer());
Vector v = null;
for (int iter = 0; iter < iterations; iter++) {
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/environment/deploy/DeployingContextImplTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/environment/deploy/DeployingContextImplTest.java
new file mode 100644
index 0000000..c51b253
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/environment/deploy/DeployingContextImplTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.ignite.ml.environment.deploy;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.apache.ignite.ml.environment.LearningEnvironment;
+import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * Tests for DeployingContextImpl.
+ */
+public class DeployingContextImplTest {
+ /** */
+ private LearningEnvironment environment;
+
+ /** */
+ @Before
+ public void setUp() throws Exception {
+ environment = LearningEnvironmentBuilder.defaultBuilder().buildForTrainer();
+ }
+
+ /** */
+ @Test
+ public void testSimpleCase() {
+ environment.initDeployingContext(new A());
+ assertEquals(A.class, environment.deployingContext().userClass());
+ assertEquals(A.class.getClassLoader(), environment.deployingContext().clientClassLoader());
+ }
+
+ /** */
+ @Test
+ public void testStraightDependency() {
+ environment.initDeployingContext(new C(new A()));
+ assertEquals(A.class, environment.deployingContext().userClass());
+ assertEquals(A.class.getClassLoader(), environment.deployingContext().clientClassLoader());
+ }
+
+ /** */
+ @Test
+ public void testNestedDependencies() {
+ environment.initDeployingContext(new C(new C(new C(new A()))));
+ assertEquals(A.class, environment.deployingContext().userClass());
+ assertEquals(A.class.getClassLoader(), environment.deployingContext().clientClassLoader());
+ }
+
+ /** */
+ @Test
+ public void testClassWithoutDependencies() {
+ environment.initDeployingContext(new C(new C(new B(new A()))));
+ assertEquals(B.class, environment.deployingContext().userClass());
+ assertEquals(B.class.getClassLoader(), environment.deployingContext().clientClassLoader());
+ }
+
+ /** */
+ @Test
+ public void testClassWithSeveralDeps1() {
+ environment.initDeployingContext(new C(new C(new D(new C(new C(new A())), new B(new A())))));
+ // in this case we should get only first dependency
+ assertEquals(A.class, environment.deployingContext().userClass());
+ assertEquals(A.class.getClassLoader(), environment.deployingContext().clientClassLoader());
+ }
+
+ /** */
+ @Test
+ public void testClassWithSeveralDeps2() {
+ environment.initDeployingContext(new C(new C(new D(new B(new A()), new C(new C(new A()))))));
+ // in this case we should get only first dependency
+ assertEquals(B.class, environment.deployingContext().userClass());
+ assertEquals(B.class.getClassLoader(), environment.deployingContext().clientClassLoader());
+ }
+
+ /** */
+ private static class A {
+
+ }
+
+ /** */
+ private static class B implements DeployableObject {
+ /** */
+ private final Object obj;
+
+ /** */
+ public B(Object obj) {
+ this.obj = obj;
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.emptyList();
+ }
+ }
+
+ /** */
+ private static class C implements DeployableObject {
+ /** */
+ private final Object obj;
+
+ /** */
+ public C(Object obj) {
+ this.obj = obj;
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Collections.singletonList(obj);
+ }
+ }
+
+ /** */
+ private static class D implements DeployableObject {
+ /** */
+ private final Object obj1;
+
+ /** */
+ private final Object obj2;
+
+ /** */
+ public D(Object obj1, Object obj2) {
+ this.obj1 = obj1;
+ this.obj2 = obj2;
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<Object> getDependencies() {
+ return Arrays.asList(obj1, obj2);
+ }
+ }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/environment/deploy/MLDeployingTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/environment/deploy/MLDeployingTest.java
new file mode 100644
index 0000000..cae422f
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/environment/deploy/MLDeployingTest.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.environment.deploy;
+
+import java.lang.reflect.Constructor;
+import java.util.Arrays;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
+import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
+import org.apache.ignite.ml.pipeline.Pipeline;
+import org.apache.ignite.ml.pipeline.PipelineMdl;
+import org.apache.ignite.ml.preprocessing.PreprocessingTrainer;
+import org.apache.ignite.ml.preprocessing.Preprocessor;
+import org.apache.ignite.ml.preprocessing.binarization.BinarizationPreprocessor;
+import org.apache.ignite.ml.preprocessing.imputing.ImputerTrainer;
+import org.apache.ignite.ml.preprocessing.minmaxscaling.MinMaxScalerTrainer;
+import org.apache.ignite.ml.preprocessing.normalization.NormalizationTrainer;
+import org.apache.ignite.ml.regressions.logistic.LogisticRegressionModel;
+import org.apache.ignite.ml.regressions.logistic.LogisticRegressionSGDTrainer;
+import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/** */
+public class MLDeployingTest extends GridCommonAbstractTest {
+ /** */
+ private static final String EXT_VECTORIZER = "org.apache.ignite.tests.p2p.ml.CustomVectorizer";
+
+ /** */
+ private static final String EXT_PREPROCESSOR_1 = "org.apache.ignite.tests.p2p.ml.CustomPreprocessor1";
+
+ /** */
+ private static final String EXT_PREPROCESSOR_2 = "org.apache.ignite.tests.p2p.ml.CustomPreprocessor2";
+
+ /** */
+ private static final int NUMBER_OF_COMPUTE_RETTRIES = 3;
+
+ /** */
+ private Ignite ignite1;
+
+ /** */
+ private Ignite ignite2;
+
+ /** */
+ private Ignite ignite3;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDiscoverySpi(new TcpDiscoverySpi()
+ .setIpFinder(new TcpDiscoveryVmIpFinder()
+ .setAddresses(Arrays.asList("127.0.0.1:47500..47509"))));
+
+ cfg.setClientMode(false);
+ cfg.setPeerClassLoadingEnabled(true);
+
+ return cfg;
+ }
+
+ /** */
+ @Before
+ public void setUp() throws Exception {
+ ignite1 = startGrid(1);
+ ignite2 = startGrid(2);
+ }
+
+ /** */
+ @After
+ public void tearDown() throws Exception {
+ stopAllGrids();
+ }
+
+ /** */
+ @Test
+ public void testCustomVectorizer() throws Exception {
+ testOnCache("testCustomVectorizer", cache -> {
+ Vectorizer<Integer, Vector, Integer, Double> vectorizer = createVectorizer();
+ fitAndTestModel(cache, vectorizer);
+ });
+ }
+
+ /** */
+ @Test
+ public void testCustomPreprocessor() throws Exception {
+ testOnCache("testCustomPreprocessor", cache -> {
+ Vectorizer<Integer, Vector, Integer, Double> vectorizer = new DummyVectorizer<>();
+ vectorizer = vectorizer.labeled(Vectorizer.LabelCoordinate.LAST);
+
+ Preprocessor<Integer, Vector> customPreprocessor1 = createPreprocessor(vectorizer, EXT_PREPROCESSOR_1);
+ Preprocessor<Integer, Vector> customPreprocessor2 = createPreprocessor(customPreprocessor1, EXT_PREPROCESSOR_2);
+
+ Preprocessor<Integer, Vector> knownPreprocessor1 = new BinarizationPreprocessor<>(0.5, customPreprocessor1);
+ Preprocessor<Integer, Vector> knownPreprocessor2 = new BinarizationPreprocessor<>(0.5, customPreprocessor2);
+
+ Preprocessor<Integer, Vector> customPreprocessor3 = createPreprocessor(knownPreprocessor2, EXT_PREPROCESSOR_1);
+ Preprocessor<Integer, Vector> customPreprocessor4 = createPreprocessor(customPreprocessor3, EXT_PREPROCESSOR_2);
+
+ fitAndTestModel(cache, customPreprocessor1);
+ fitAndTestModel(cache, customPreprocessor2);
+ fitAndTestModel(cache, knownPreprocessor1);
+ fitAndTestModel(cache, knownPreprocessor2);
+ fitAndTestModel(cache, customPreprocessor3);
+ fitAndTestModel(cache, customPreprocessor4);
+ });
+ }
+
+ /** */
+ @Test
+ public void testPipeline() throws Exception {
+ testOnCache("testPipeline", cache -> {
+ Vectorizer<Integer, Vector, Integer, Double> vectorizer = createVectorizer();
+
+ PipelineMdl<Integer, Vector> mdl = new Pipeline<Integer, Vector, Integer, Double>()
+ .addVectorizer(vectorizer)
+ .addPreprocessingTrainer(new ImputerTrainer<Integer, Vector>())
+ .addPreprocessingTrainer(makePreprocessorTrainer(EXT_PREPROCESSOR_2))
+ .addPreprocessingTrainer(new MinMaxScalerTrainer<Integer, Vector>())
+ .addPreprocessingTrainer(makePreprocessorTrainer(EXT_PREPROCESSOR_1))
+ .addPreprocessingTrainer(new NormalizationTrainer<Integer, Vector>()
+ .withP(1))
+ .addTrainer(new DecisionTreeClassificationTrainer(5, 0))
+ .fit(cache);
+
+ assertEquals(0., mdl.predict(VectorUtils.of(0., 0.)), 1.);
+ });
+ }
+
+ /** */
+ private void testOnCache(String cacheName, TestCacheConsumer body) throws Exception {
+ IgniteCache<Integer, Vector> cache = null;
+ try {
+ cache = prepareCache(ignite1, cacheName);
+ body.accept(new CacheBasedDatasetBuilder<>(ignite1, cache)
+ .withRetriesNumber(NUMBER_OF_COMPUTE_RETTRIES));
+ } finally {
+ if(cache != null)
+ cache.destroy();
+ }
+ }
+
+ /** */
+ private void fitAndTestModel(CacheBasedDatasetBuilder<Integer, Vector> datasetBuilder,
+ Preprocessor<Integer, Vector> preprocessor) {
+ LogisticRegressionSGDTrainer trainer = new LogisticRegressionSGDTrainer();
+ LogisticRegressionModel model = trainer.fit(datasetBuilder, preprocessor);
+
+ // For this case any answer is valid.
+ assertEquals(0., model.predict(VectorUtils.of(0., 0.)), 1.);
+ }
+
+ /** */
+ private Vectorizer<Integer, Vector, Integer, Double> createVectorizer() throws ClassNotFoundException, NoSuchMethodException, InstantiationException, IllegalAccessException, java.lang.reflect.InvocationTargetException {
+ ClassLoader loader = getExternalClassLoader();
+ Class<?> clazz = loader.loadClass(EXT_VECTORIZER);
+
+ Constructor ctor = clazz.getConstructor();
+ Vectorizer<Integer, Vector, Integer, Double> vectorizer =
+ (Vectorizer<Integer, Vector, Integer, Double>)ctor.newInstance();
+ vectorizer = vectorizer.labeled(Vectorizer.LabelCoordinate.LAST);
+ return vectorizer;
+ }
+
+ /** */
+ private Preprocessor<Integer, Vector> createPreprocessor(Preprocessor<Integer, Vector> basePreprocessor,
+ String className) throws Exception {
+ ClassLoader loader = getExternalClassLoader();
+ Class<?> clazz = loader.loadClass(className);
+
+ Constructor ctor = clazz.getConstructor(Preprocessor.class);
+ return (Preprocessor<Integer, Vector>)ctor.newInstance(basePreprocessor);
+ }
+
+ /** */
+ @NotNull private PreprocessingTrainer makePreprocessorTrainer(String preprocessorClassName) throws Exception {
+ return new PreprocessingTrainer() {
+ @Override
+ public Preprocessor fit(LearningEnvironmentBuilder envBuilder, DatasetBuilder datasetBuilder,
+ Preprocessor basePreprocessor) {
+ try {
+ return createPreprocessor(basePreprocessor, preprocessorClassName);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+
+ /** xor truth table. */
+ private static final double[][] xor = {
+ {0.0, 0.0, 0.0},
+ {0.0, 1.0, 1.0},
+ {1.0, 0.0, 1.0},
+ {1.0, 1.0, 0.0}
+ };
+
+ /** */
+ private IgniteCache<Integer, Vector> prepareCache(Ignite ignite, String cacheName) {
+ IgniteCache<Integer, Vector> cache = ignite.getOrCreateCache(new CacheConfiguration<Integer, Vector>(cacheName));
+
+ for (int i = 0; i < xor.length; i++)
+ cache.put(i, VectorUtils.of(xor[i]));
+
+ return cache;
+ }
+
+ /** */
+ @FunctionalInterface
+ private static interface TestCacheConsumer {
+ public void accept(CacheBasedDatasetBuilder<Integer, Vector> datasetBuilder) throws Exception;
+ }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeapTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeapTest.java
index 91e96bc..abcff2c 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeapTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeapTest.java
@@ -62,7 +62,8 @@ public class LSQROnHeapTest extends TrainerTest {
LSQROnHeap<Integer, Vector> lsqr = new LSQROnHeap<>(
datasetBuilder,
TestUtils.testEnvBuilder(),
- new SimpleLabeledDatasetDataBuilder<>(prerocessor)
+ new SimpleLabeledDatasetDataBuilder<>(prerocessor),
+ TestUtils.testEnvBuilder().buildForTrainer()
);
LSQRResult res = lsqr.solve(0, 1e-12, 1e-12, 1e8, -1, false, null);
@@ -100,7 +101,8 @@ public class LSQROnHeapTest extends TrainerTest {
LSQROnHeap<Integer, Vector> lsqr = new LSQROnHeap<>(
datasetBuilder,
TestUtils.testEnvBuilder(),
- new SimpleLabeledDatasetDataBuilder<>(prerocessor)
+ new SimpleLabeledDatasetDataBuilder<>(prerocessor),
+ TestUtils.testEnvBuilder().buildForTrainer()
);
LSQRResult res = lsqr.solve(0, 1e-12, 1e-12, 1e8, -1, false,
@@ -138,7 +140,7 @@ public class LSQROnHeapTest extends TrainerTest {
try (LSQROnHeap<Integer, Vector> lsqr = new LSQROnHeap<>(
datasetBuilder,
TestUtils.testEnvBuilder(),
- new SimpleLabeledDatasetDataBuilder<>(prerocessor)))
+ new SimpleLabeledDatasetDataBuilder<>(prerocessor), TestUtils.testEnvBuilder().buildForTrainer()))
{
LSQRResult res = lsqr.solve(0, 1e-12, 1e-12, 1e8, -1, false, null);
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/util/generators/DataStreamGeneratorFillCacheTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/util/generators/DataStreamGeneratorFillCacheTest.java
index 8e2787e..e2608a8 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/util/generators/DataStreamGeneratorFillCacheTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/util/generators/DataStreamGeneratorFillCacheTest.java
@@ -17,6 +17,9 @@
package org.apache.ignite.ml.util.generators;
+import java.util.Arrays;
+import java.util.UUID;
+import java.util.stream.DoubleStream;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
@@ -30,40 +33,28 @@ import org.apache.ignite.ml.dataset.primitive.builder.context.EmptyContextBuilde
import org.apache.ignite.ml.dataset.primitive.builder.data.SimpleDatasetDataBuilder;
import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
import org.apache.ignite.ml.dataset.primitive.data.SimpleDatasetData;
+import org.apache.ignite.ml.environment.LearningEnvironment;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.ml.structures.LabeledVector;
import org.apache.ignite.ml.util.generators.primitives.scalar.GaussRandomProducer;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.junit.After;
-import org.junit.Before;
import org.junit.Test;
-import java.util.UUID;
-import java.util.stream.DoubleStream;
-
/**
* Test for {@link DataStreamGenerator} cache filling.
*/
public class DataStreamGeneratorFillCacheTest extends GridCommonAbstractTest {
/** */
- private Ignite ignite;
-
- /** */
- @Before
- public void before() throws Exception {
- ignite = startGrid();
- }
-
- /** */
- @After
- public void after() throws Exception {
- ignite.close();
- }
-
- /** */
@Test
public void testCacheFilling() {
- IgniteConfiguration configuration = new IgniteConfiguration().setPeerClassLoadingEnabled(true);
+ IgniteConfiguration configuration = new IgniteConfiguration()
+ .setDiscoverySpi(new TcpDiscoverySpi()
+ .setIpFinder(new TcpDiscoveryVmIpFinder()
+ .setAddresses(Arrays.asList("127.0.0.1:47500..47509"))));
+
String cacheName = "TEST_CACHE";
CacheConfiguration<UUID, LabeledVector<Double>> cacheConfiguration =
new CacheConfiguration<UUID, LabeledVector<Double>>(cacheName)
@@ -77,15 +68,21 @@ public class DataStreamGeneratorFillCacheTest extends GridCommonAbstractTest {
LabeledDummyVectorizer<UUID, Double> vectorizer = new LabeledDummyVectorizer<>();
CacheBasedDatasetBuilder<UUID, LabeledVector<Double>> datasetBuilder = new CacheBasedDatasetBuilder<>(ignite, cache);
- try (CacheBasedDataset<UUID, LabeledVector<Double>, EmptyContext, SimpleDatasetData> dataset =
- datasetBuilder.build(LearningEnvironmentBuilder.defaultBuilder(),
- new EmptyContextBuilder<>(), new SimpleDatasetDataBuilder<>(vectorizer))) {
- StatPair result = dataset.compute(
- data -> new StatPair(DoubleStream.of(data.getFeatures()).sum(), data.getRows()),
- StatPair::sum
- );
+ IgniteFunction<SimpleDatasetData, StatPair> map = data ->
+ new StatPair(DoubleStream.of(data.getFeatures()).sum(), data.getRows());
+ LearningEnvironment env = LearningEnvironmentBuilder.defaultBuilder().buildForTrainer();
+ env.deployingContext().initByClientObject(map);
+ try (CacheBasedDataset<UUID, LabeledVector<Double>, EmptyContext, SimpleDatasetData> dataset =
+ datasetBuilder.build(
+ LearningEnvironmentBuilder.defaultBuilder(),
+ new EmptyContextBuilder<>(),
+ new SimpleDatasetDataBuilder<>(vectorizer),
+ env
+ )) {
+
+ StatPair result = dataset.compute(map, StatPair::sum);
assertEquals(datasetSize, result.countOfRows);
assertEquals(0.0, result.elementsSum / result.countOfRows, 1e-2);
}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/util/generators/DataStreamGeneratorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/util/generators/DataStreamGeneratorTest.java
index d24e59f..3381018 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/util/generators/DataStreamGeneratorTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/util/generators/DataStreamGeneratorTest.java
@@ -176,7 +176,8 @@ public class DataStreamGeneratorTest {
DatasetBuilder<Vector, Double> b1) {
return b1.build(LearningEnvironmentBuilder.defaultBuilder(),
new EmptyContextBuilder<>(),
- new LabeledDatasetPartitionDataBuilderOnHeap<>((Preprocessor<Vector, Double>)LabeledVector::new)
+ new LabeledDatasetPartitionDataBuilderOnHeap<>((Preprocessor<Vector, Double>)LabeledVector::new),
+ LearningEnvironmentBuilder.defaultBuilder().buildForTrainer()
);
}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowServerAddressSpec.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowServerAddressSpec.java
index 727a030..9421874 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowServerAddressSpec.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowServerAddressSpec.java
@@ -57,7 +57,6 @@ public class TensorFlowServerAddressSpec implements Serializable {
*/
public String format(Ignite ignite) {
Collection<String> names = ignite.cluster().forNodeId(nodeId).hostNames();
-
return names.iterator().next() + ":" + port;
}